From 91f2244ff68c5a0682aa2520831275004195182d Mon Sep 17 00:00:00 2001 From: Brian Cipriano Date: Mon, 12 Dec 2022 14:30:07 -0500 Subject: [PATCH 01/56] Upgrade Log4j to 2.16. (#1080) --- cuebot/build.gradle | 5 +- .../spring/remoting/CueServerInterceptor.java | 7 +- .../common/spring/remoting/GrpcServer.java | 5 +- .../imageworks/spcue/CuebotApplication.java | 5 +- .../com/imageworks/spcue/SortableShow.java | 5 +- .../dao/criteria/postgres/FrameSearch.java | 5 +- .../spcue/dao/postgres/DispatcherDaoJdbc.java | 5 +- .../spcue/dao/postgres/ProcDaoJdbc.java | 2 - .../spcue/dao/postgres/WhiteboardDaoJdbc.java | 5 +- .../spcue/dispatcher/AbstractDispatcher.java | 5 +- .../spcue/dispatcher/BookingQueue.java | 5 +- .../spcue/dispatcher/CoreUnitDispatcher.java | 5 +- .../spcue/dispatcher/DispatchQueue.java | 5 +- .../dispatcher/DispatchSupportService.java | 5 +- .../dispatcher/FrameCompleteHandler.java | 6 +- .../spcue/dispatcher/HealthyThreadPool.java | 5 +- .../spcue/dispatcher/HostReportHandler.java | 5 +- .../spcue/dispatcher/HostReportQueue.java | 5 +- .../spcue/dispatcher/LocalDispatcher.java | 5 +- .../spcue/dispatcher/RedirectManager.java | 5 +- .../ThreadPoolTaskExecutorWrapper.java | 5 +- .../dispatcher/commands/DispatchBookHost.java | 5 +- .../commands/DispatchRqdKillFrame.java | 5 +- .../imageworks/spcue/rqd/RqdClientGrpc.java | 5 +- .../spcue/servant/ManageDepend.java | 5 +- .../imageworks/spcue/servant/ManageJob.java | 5 +- .../spcue/service/AdminManagerService.java | 5 +- .../spcue/service/BookingManagerService.java | 5 +- .../spcue/service/DependManagerService.java | 5 +- .../spcue/service/EmailSupport.java | 5 +- .../spcue/service/FilterManagerService.java | 5 +- .../spcue/service/HistoricalSupport.java | 5 +- .../spcue/service/HostManagerService.java | 5 +- .../imageworks/spcue/service/JmsMover.java | 5 +- .../imageworks/spcue/service/JobLauncher.java | 5 +- .../spcue/service/JobManagerService.java | 5 +- .../spcue/service/JobManagerSupport.java | 5 +- .../com/imageworks/spcue/service/JobSpec.java | 5 +- .../spcue/service/LocalBookingSupport.java | 5 +- .../service/MaintenanceManagerSupport.java | 5 +- .../spcue/service/RedirectService.java | 5 +- .../spcue/service/WhiteboardService.java | 5 +- .../spcue/servlet/JobLaunchServlet.java | 5 +- .../spcue/util/CueExceptionUtil.java | 5 +- .../com/imageworks/spcue/util/CueUtil.java | 5 +- cuebot/src/main/resources/log4j.properties | 51 --------- cuebot/src/main/resources/log4j2.properties | 103 ++++++++++++++++++ 47 files changed, 236 insertions(+), 143 deletions(-) delete mode 100644 cuebot/src/main/resources/log4j.properties create mode 100644 cuebot/src/main/resources/log4j2.properties diff --git a/cuebot/build.gradle b/cuebot/build.gradle index fe5abf3e2..a9b7cfaa5 100644 --- a/cuebot/build.gradle +++ b/cuebot/build.gradle @@ -55,8 +55,9 @@ dependencies { compile group: 'org.quartz-scheduler', name: 'quartz', version: '2.2.1', { exclude group: 'c3p0', module: 'c3p0' } compile group: 'org.postgresql', name: 'postgresql', version: '42.2.2' compile group: 'com.google.protobuf', name: 'protobuf-java', version: '3.21.2' - compile group: 'log4j', name: 'log4j', version: '1.2.17' - compile group: 'org.slf4j', name: 'slf4j-log4j12', version: '1.7.26' + compile group: 'org.apache.logging.log4j', name: 'log4j-api', version: '2.16.0' + compile group: 'org.apache.logging.log4j', name: 'log4j-core', version: '2.16.0' + compile group: 'org.apache.logging.log4j', name: 'log4j-slf4j-impl', version: '2.16.0' protobuf fileTree("../proto/") diff --git a/cuebot/src/main/java/com/imageworks/common/spring/remoting/CueServerInterceptor.java b/cuebot/src/main/java/com/imageworks/common/spring/remoting/CueServerInterceptor.java index 69c448b11..31ebeb12e 100644 --- a/cuebot/src/main/java/com/imageworks/common/spring/remoting/CueServerInterceptor.java +++ b/cuebot/src/main/java/com/imageworks/common/spring/remoting/CueServerInterceptor.java @@ -7,13 +7,14 @@ import io.grpc.ServerCallHandler; import io.grpc.ServerInterceptor; import io.grpc.Status; -import org.apache.log4j.Logger; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.LogManager; public class CueServerInterceptor implements ServerInterceptor { - private static final Logger logger = Logger.getLogger(CueServerInterceptor.class); - private static final Logger accessLogger = Logger.getLogger("API"); + private static final Logger logger = LogManager.getLogger(CueServerInterceptor.class); + private static final Logger accessLogger = LogManager.getLogger("API"); @Override public ServerCall.Listener interceptCall( diff --git a/cuebot/src/main/java/com/imageworks/common/spring/remoting/GrpcServer.java b/cuebot/src/main/java/com/imageworks/common/spring/remoting/GrpcServer.java index 37c1c408d..a5038f82c 100644 --- a/cuebot/src/main/java/com/imageworks/common/spring/remoting/GrpcServer.java +++ b/cuebot/src/main/java/com/imageworks/common/spring/remoting/GrpcServer.java @@ -6,7 +6,8 @@ import io.grpc.Server; import io.grpc.ServerBuilder; -import org.apache.log4j.Logger; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.LogManager; import org.springframework.beans.BeansException; import org.springframework.context.ApplicationContext; import org.springframework.context.ApplicationContextAware; @@ -40,7 +41,7 @@ public class GrpcServer implements ApplicationContextAware { - private static final Logger logger = Logger.getLogger(GrpcServer.class); + private static final Logger logger = LogManager.getLogger(GrpcServer.class); private static final String DEFAULT_NAME = "CueGrpcServer"; private static final String DEFAULT_PORT = "8443"; diff --git a/cuebot/src/main/java/com/imageworks/spcue/CuebotApplication.java b/cuebot/src/main/java/com/imageworks/spcue/CuebotApplication.java index 12087f93e..6ef64080c 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/CuebotApplication.java +++ b/cuebot/src/main/java/com/imageworks/spcue/CuebotApplication.java @@ -23,7 +23,8 @@ import java.util.stream.Stream; import org.apache.commons.lang.StringUtils; -import org.apache.log4j.Logger; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.LogManager; import org.springframework.boot.SpringApplication; import org.springframework.boot.autoconfigure.SpringBootApplication; @@ -34,7 +35,7 @@ private static String[] checkArgs(String[] args) { .filter(arg -> arg.startsWith("--log.frame-log-root=")).findFirst(); if (deprecatedFlag.isPresent()) { // Log a deprecation warning. - Logger warning_logger = Logger.getLogger(CuebotApplication.class); + Logger warning_logger = LogManager.getLogger(CuebotApplication.class); warning_logger.warn("`--log.frame-log-root` is deprecated and will be removed in an " + "upcoming release. It has been replaced with `--log.frame-log-root.default_os`. " + "See opencue.properties for details on OpenCue's new OS-dependent root directories."); diff --git a/cuebot/src/main/java/com/imageworks/spcue/SortableShow.java b/cuebot/src/main/java/com/imageworks/spcue/SortableShow.java index 83fe7da5d..f13fbaae2 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/SortableShow.java +++ b/cuebot/src/main/java/com/imageworks/spcue/SortableShow.java @@ -24,11 +24,12 @@ import java.util.Set; import java.util.concurrent.ConcurrentHashMap; -import org.apache.log4j.Logger; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.LogManager; public class SortableShow implements Comparable { - private static final Logger logger = Logger.getLogger(SortableShow.class); + private static final Logger logger = LogManager.getLogger(SortableShow.class); private String show; private float tier; diff --git a/cuebot/src/main/java/com/imageworks/spcue/dao/criteria/postgres/FrameSearch.java b/cuebot/src/main/java/com/imageworks/spcue/dao/criteria/postgres/FrameSearch.java index ee3a1f841..de33e29cd 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dao/criteria/postgres/FrameSearch.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dao/criteria/postgres/FrameSearch.java @@ -23,7 +23,8 @@ import java.util.stream.Collectors; import com.google.common.collect.ImmutableList; -import org.apache.log4j.Logger; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.LogManager; import com.imageworks.spcue.FrameInterface; import com.imageworks.spcue.JobInterface; @@ -36,7 +37,7 @@ public class FrameSearch extends Criteria implements FrameSearchInterface { private static final int MAX_RESULTS = 1000; - private static final Logger logger = Logger.getLogger(FrameSearch.class); + private static final Logger logger = LogManager.getLogger(FrameSearch.class); private static final Pattern PATTERN_SINGLE_FRAME = Pattern.compile("^([0-9]+)$"); private static final Pattern PATTERN_RANGE = Pattern.compile("^([0-9]+)\\-([0-9]+)$"); private static final Pattern PATTERN_FLOAT_RANGE = Pattern.compile("^([0-9\\.]+)\\-([0-9\\.]+)$"); diff --git a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/DispatcherDaoJdbc.java b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/DispatcherDaoJdbc.java index 4c9570d9a..032c90dac 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/DispatcherDaoJdbc.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/DispatcherDaoJdbc.java @@ -28,7 +28,8 @@ import java.util.Set; import java.util.concurrent.ConcurrentHashMap; -import org.apache.log4j.Logger; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.LogManager; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.core.env.Environment; import org.springframework.jdbc.core.RowMapper; @@ -58,7 +59,7 @@ */ public class DispatcherDaoJdbc extends JdbcDaoSupport implements DispatcherDao { - private static final Logger logger = Logger.getLogger(DispatcherDaoJdbc.class); + private static final Logger logger = LogManager.getLogger(DispatcherDaoJdbc.class); public static final RowMapper PKJOB_MAPPER = new RowMapper() { diff --git a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/ProcDaoJdbc.java b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/ProcDaoJdbc.java index 32c454e0f..b68f8a74c 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/ProcDaoJdbc.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/ProcDaoJdbc.java @@ -29,8 +29,6 @@ import java.util.List; import java.util.Map; -import com.imageworks.spcue.dispatcher.AbstractDispatcher; -import org.apache.log4j.Logger; import org.springframework.dao.DataAccessException; import org.springframework.jdbc.core.RowMapper; import org.springframework.jdbc.core.support.JdbcDaoSupport; diff --git a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/WhiteboardDaoJdbc.java b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/WhiteboardDaoJdbc.java index 9ffe56beb..38566470a 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/WhiteboardDaoJdbc.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/WhiteboardDaoJdbc.java @@ -29,7 +29,8 @@ import com.google.common.collect.Lists; import com.google.common.collect.Sets; import com.google.protobuf.ByteString; -import org.apache.log4j.Logger; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.LogManager; import org.springframework.jdbc.core.RowMapper; import org.springframework.jdbc.core.support.JdbcDaoSupport; @@ -129,7 +130,7 @@ public class WhiteboardDaoJdbc extends JdbcDaoSupport implements WhiteboardDao { @SuppressWarnings("unused") - private static final Logger logger = Logger.getLogger(WhiteboardDaoJdbc.class); + private static final Logger logger = LogManager.getLogger(WhiteboardDaoJdbc.class); private FrameSearchFactory frameSearchFactory; private ProcSearchFactory procSearchFactory; diff --git a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/AbstractDispatcher.java b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/AbstractDispatcher.java index 73f5aef73..355c64175 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/AbstractDispatcher.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/AbstractDispatcher.java @@ -19,7 +19,8 @@ package com.imageworks.spcue.dispatcher; -import org.apache.log4j.Logger; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.LogManager; import com.imageworks.spcue.DispatchFrame; import com.imageworks.spcue.VirtualProc; @@ -33,7 +34,7 @@ */ public abstract class AbstractDispatcher { - private static final Logger logger = Logger.getLogger(AbstractDispatcher.class); + private static final Logger logger = LogManager.getLogger(AbstractDispatcher.class); public DispatchSupport dispatchSupport; public RqdClient rqdClient; diff --git a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/BookingQueue.java b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/BookingQueue.java index 347acd025..69a961977 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/BookingQueue.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/BookingQueue.java @@ -23,7 +23,8 @@ import com.google.common.cache.CacheBuilder; import com.imageworks.spcue.dispatcher.commands.KeyRunnable; -import org.apache.log4j.Logger; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.LogManager; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.core.env.Environment; @@ -36,7 +37,7 @@ public class BookingQueue { private final int maxPoolSize; private static final int BASE_SLEEP_TIME_MILLIS = 300; - private static final Logger logger = Logger.getLogger("HEALTH"); + private static final Logger logger = LogManager.getLogger("HEALTH"); private HealthyThreadPool healthyThreadPool; public BookingQueue(int healthThreshold, int minUnhealthyPeriodMin, int queueCapacity, diff --git a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/CoreUnitDispatcher.java b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/CoreUnitDispatcher.java index 4d556589e..a9a8b918a 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/CoreUnitDispatcher.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/CoreUnitDispatcher.java @@ -25,7 +25,8 @@ import com.google.common.cache.Cache; import com.google.common.cache.CacheBuilder; -import org.apache.log4j.Logger; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.LogManager; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.core.env.Environment; @@ -85,7 +86,7 @@ */ public class CoreUnitDispatcher implements Dispatcher { private static final Logger logger = - Logger.getLogger(CoreUnitDispatcher.class); + LogManager.getLogger(CoreUnitDispatcher.class); private DispatchSupport dispatchSupport; diff --git a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/DispatchQueue.java b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/DispatchQueue.java index 3798bddaa..00e552a05 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/DispatchQueue.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/DispatchQueue.java @@ -22,7 +22,8 @@ import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicLong; -import org.apache.log4j.Logger; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.LogManager; import com.imageworks.spcue.dispatcher.commands.KeyRunnable; public class DispatchQueue { @@ -33,7 +34,7 @@ public class DispatchQueue { private int corePoolSize; private int maxPoolSize; - private static final Logger logger = Logger.getLogger("HEALTH"); + private static final Logger logger = LogManager.getLogger("HEALTH"); private String name = "Default"; private HealthyThreadPool healthyDispatchPool; diff --git a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/DispatchSupportService.java b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/DispatchSupportService.java index dd8f71cf0..c935024d6 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/DispatchSupportService.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/DispatchSupportService.java @@ -39,7 +39,8 @@ import com.imageworks.spcue.ShowInterface; import com.imageworks.spcue.StrandedCores; import com.imageworks.spcue.VirtualProc; -import org.apache.log4j.Logger; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.LogManager; import org.springframework.dao.EmptyResultDataAccessException; import org.springframework.transaction.annotation.Propagation; import org.springframework.transaction.annotation.Transactional; @@ -64,7 +65,7 @@ @Transactional(propagation = Propagation.REQUIRED) public class DispatchSupportService implements DispatchSupport { - private static final Logger logger = Logger.getLogger(DispatchSupportService.class); + private static final Logger logger = LogManager.getLogger(DispatchSupportService.class); private JobDao jobDao; private FrameDao frameDao; diff --git a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/FrameCompleteHandler.java b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/FrameCompleteHandler.java index d4e619894..55336aaf4 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/FrameCompleteHandler.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/FrameCompleteHandler.java @@ -24,8 +24,8 @@ import java.util.Random; import java.util.concurrent.atomic.AtomicLong; -import com.imageworks.spcue.*; -import org.apache.log4j.Logger; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.LogManager; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.core.env.Environment; import org.springframework.dao.EmptyResultDataAccessException; @@ -66,7 +66,7 @@ */ public class FrameCompleteHandler { - private static final Logger logger = Logger.getLogger(FrameCompleteHandler.class); + private static final Logger logger = LogManager.getLogger(FrameCompleteHandler.class); private static final Random randomNumber = new Random(); diff --git a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/HealthyThreadPool.java b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/HealthyThreadPool.java index 13c96e776..5c2efabc4 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/HealthyThreadPool.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/HealthyThreadPool.java @@ -13,7 +13,8 @@ import com.google.common.cache.CacheBuilder; import com.imageworks.spcue.dispatcher.commands.DispatchBookHost; import com.imageworks.spcue.dispatcher.commands.KeyRunnable; -import org.apache.log4j.Logger; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.LogManager; /*** @@ -24,7 +25,7 @@ */ public class HealthyThreadPool extends ThreadPoolExecutor { // The service need s to be unhealthy for this period of time to report - private static final Logger logger = Logger.getLogger("HEALTH"); + private static final Logger logger = LogManager.getLogger("HEALTH"); // Threshold to consider healthy or unhealthy private final int healthThreshold; private final int poolSize; diff --git a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/HostReportHandler.java b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/HostReportHandler.java index a63da07e8..d763cce53 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/HostReportHandler.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/HostReportHandler.java @@ -27,7 +27,8 @@ import java.util.Map; import java.util.concurrent.ThreadPoolExecutor; -import org.apache.log4j.Logger; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.LogManager; import org.springframework.core.task.TaskRejectedException; import org.springframework.dao.DataAccessException; import org.springframework.dao.EmptyResultDataAccessException; @@ -64,7 +65,7 @@ public class HostReportHandler { - private static final Logger logger = Logger.getLogger(HostReportHandler.class); + private static final Logger logger = LogManager.getLogger(HostReportHandler.class); private BookingManager bookingManager; private HostManager hostManager; diff --git a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/HostReportQueue.java b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/HostReportQueue.java index 59b91abcc..6326086ae 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/HostReportQueue.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/HostReportQueue.java @@ -26,16 +26,17 @@ import java.util.concurrent.atomic.AtomicBoolean; import com.imageworks.spcue.grpc.report.HostReport; -import org.apache.log4j.Logger; import com.google.common.cache.Cache; import com.google.common.cache.CacheBuilder; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.LogManager; import com.imageworks.spcue.dispatcher.commands.DispatchHandleHostReport; import com.imageworks.spcue.util.CueUtil; public class HostReportQueue extends ThreadPoolExecutor { - private static final Logger logger = Logger.getLogger(HostReportQueue.class); + private static final Logger logger = LogManager.getLogger(HostReportQueue.class); private QueueRejectCounter rejectCounter = new QueueRejectCounter(); private AtomicBoolean isShutdown = new AtomicBoolean(false); private int queueCapacity; diff --git a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/LocalDispatcher.java b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/LocalDispatcher.java index 23bf6f73a..288965a04 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/LocalDispatcher.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/LocalDispatcher.java @@ -22,7 +22,8 @@ import java.util.ArrayList; import java.util.List; -import org.apache.log4j.Logger; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.LogManager; import org.springframework.dao.EmptyResultDataAccessException; import com.imageworks.spcue.DispatchFrame; @@ -42,7 +43,7 @@ public class LocalDispatcher extends AbstractDispatcher implements Dispatcher { private static final Logger logger = - Logger.getLogger(LocalDispatcher.class); + LogManager.getLogger(LocalDispatcher.class); private BookingManager bookingManager; private JobManager jobManager; diff --git a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/RedirectManager.java b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/RedirectManager.java index 24b1681e9..e665345cd 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/RedirectManager.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/RedirectManager.java @@ -21,7 +21,8 @@ import java.util.ArrayList; import java.util.List; -import org.apache.log4j.Logger; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.LogManager; import com.imageworks.spcue.DispatchHost; import com.imageworks.spcue.GroupInterface; @@ -46,7 +47,7 @@ public class RedirectManager { - private static final Logger logger = Logger.getLogger(RedirectManager.class); + private static final Logger logger = LogManager.getLogger(RedirectManager.class); private JobDao jobDao; private ProcDao procDao; diff --git a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/ThreadPoolTaskExecutorWrapper.java b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/ThreadPoolTaskExecutorWrapper.java index 8b749cdb1..0090d3619 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/ThreadPoolTaskExecutorWrapper.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/ThreadPoolTaskExecutorWrapper.java @@ -19,7 +19,8 @@ package com.imageworks.spcue.dispatcher; -import org.apache.log4j.Logger; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.LogManager; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.core.env.Environment; import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor; @@ -31,7 +32,7 @@ */ public class ThreadPoolTaskExecutorWrapper extends ThreadPoolTaskExecutor { - private static final Logger logger = Logger.getLogger(ThreadPoolTaskExecutorWrapper.class); + private static final Logger logger = LogManager.getLogger(ThreadPoolTaskExecutorWrapper.class); private static final long serialVersionUID = -2977068663355369141L; private int queueCapacity; diff --git a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/commands/DispatchBookHost.java b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/commands/DispatchBookHost.java index 0b65257dd..64329fc0f 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/commands/DispatchBookHost.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/commands/DispatchBookHost.java @@ -21,7 +21,8 @@ import java.util.List; import java.util.ArrayList; -import org.apache.log4j.Logger; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.LogManager; import com.imageworks.spcue.DispatchHost; import com.imageworks.spcue.GroupInterface; @@ -37,7 +38,7 @@ */ public class DispatchBookHost extends KeyRunnable { private static final Logger logger = - Logger.getLogger(DispatchBookHost.class); + LogManager.getLogger(DispatchBookHost.class); private ShowInterface show = null; private GroupInterface group = null; diff --git a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/commands/DispatchRqdKillFrame.java b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/commands/DispatchRqdKillFrame.java index 3a43c6fc3..61258a824 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/commands/DispatchRqdKillFrame.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/commands/DispatchRqdKillFrame.java @@ -19,7 +19,8 @@ package com.imageworks.spcue.dispatcher.commands; -import org.apache.log4j.Logger; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.LogManager; import com.imageworks.spcue.dispatcher.commands.KeyRunnable; import com.imageworks.spcue.VirtualProc; @@ -28,7 +29,7 @@ public class DispatchRqdKillFrame extends KeyRunnable { - private static final Logger logger = Logger.getLogger(DispatchRqdKillFrame.class); + private static final Logger logger = LogManager.getLogger(DispatchRqdKillFrame.class); private VirtualProc proc = null; private String message; diff --git a/cuebot/src/main/java/com/imageworks/spcue/rqd/RqdClientGrpc.java b/cuebot/src/main/java/com/imageworks/spcue/rqd/RqdClientGrpc.java index 5e1e016fa..40554904b 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/rqd/RqdClientGrpc.java +++ b/cuebot/src/main/java/com/imageworks/spcue/rqd/RqdClientGrpc.java @@ -23,7 +23,8 @@ import io.grpc.ManagedChannel; import io.grpc.ManagedChannelBuilder; import io.grpc.StatusRuntimeException; -import org.apache.log4j.Logger; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.LogManager; import com.google.common.cache.CacheBuilder; import com.google.common.cache.CacheLoader; @@ -49,7 +50,7 @@ import com.imageworks.spcue.grpc.rqd.RunningFrameStatusResponse; public final class RqdClientGrpc implements RqdClient { - private static final Logger logger = Logger.getLogger(RqdClientGrpc.class); + private static final Logger logger = LogManager.getLogger(RqdClientGrpc.class); private final int rqdCacheSize; private final int rqdCacheExpiration; diff --git a/cuebot/src/main/java/com/imageworks/spcue/servant/ManageDepend.java b/cuebot/src/main/java/com/imageworks/spcue/servant/ManageDepend.java index 5d1c2ab6e..9fad37ef1 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/servant/ManageDepend.java +++ b/cuebot/src/main/java/com/imageworks/spcue/servant/ManageDepend.java @@ -21,7 +21,8 @@ import io.grpc.Status; import io.grpc.stub.StreamObserver; -import org.apache.log4j.Logger; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.LogManager; import org.springframework.dao.EmptyResultDataAccessException; import com.imageworks.spcue.LightweightDependency; @@ -39,7 +40,7 @@ public class ManageDepend extends DependInterfaceGrpc.DependInterfaceImplBase { - private static final Logger logger = Logger.getLogger(ManageDepend.class); + private static final Logger logger = LogManager.getLogger(ManageDepend.class); private DependManager dependManager; private DispatchQueue manageQueue; diff --git a/cuebot/src/main/java/com/imageworks/spcue/servant/ManageJob.java b/cuebot/src/main/java/com/imageworks/spcue/servant/ManageJob.java index 01601998a..e3cfa5178 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/servant/ManageJob.java +++ b/cuebot/src/main/java/com/imageworks/spcue/servant/ManageJob.java @@ -24,7 +24,8 @@ import io.grpc.Status; import io.grpc.stub.StreamObserver; -import org.apache.log4j.Logger; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.LogManager; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.core.env.Environment; import org.springframework.dao.EmptyResultDataAccessException; @@ -161,7 +162,7 @@ import static com.imageworks.spcue.servant.ServantUtil.attemptChange; public class ManageJob extends JobInterfaceGrpc.JobInterfaceImplBase { - private static final Logger logger = Logger.getLogger(ManageJob.class); + private static final Logger logger = LogManager.getLogger(ManageJob.class); private Whiteboard whiteboard; private JobManager jobManager; private GroupManager groupManager; diff --git a/cuebot/src/main/java/com/imageworks/spcue/service/AdminManagerService.java b/cuebot/src/main/java/com/imageworks/spcue/service/AdminManagerService.java index a824dfb09..8f1f66133 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/service/AdminManagerService.java +++ b/cuebot/src/main/java/com/imageworks/spcue/service/AdminManagerService.java @@ -19,7 +19,8 @@ package com.imageworks.spcue.service; -import org.apache.log4j.Logger; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.LogManager; import org.springframework.transaction.annotation.Propagation; import org.springframework.transaction.annotation.Transactional; @@ -46,7 +47,7 @@ public class AdminManagerService implements AdminManager { @SuppressWarnings("unused") - private static final Logger logger = Logger.getLogger(AdminManagerService.class); + private static final Logger logger = LogManager.getLogger(AdminManagerService.class); private ShowDao showDao; diff --git a/cuebot/src/main/java/com/imageworks/spcue/service/BookingManagerService.java b/cuebot/src/main/java/com/imageworks/spcue/service/BookingManagerService.java index 1a2b6cee2..1322b622d 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/service/BookingManagerService.java +++ b/cuebot/src/main/java/com/imageworks/spcue/service/BookingManagerService.java @@ -21,7 +21,8 @@ import java.util.List; -import org.apache.log4j.Logger; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.LogManager; import org.springframework.dao.EmptyResultDataAccessException; import org.springframework.transaction.annotation.Propagation; import org.springframework.transaction.annotation.Transactional; @@ -49,7 +50,7 @@ public class BookingManagerService implements BookingManager { @SuppressWarnings("unused") private static final Logger logger = - Logger.getLogger(BookingManagerService.class); + LogManager.getLogger(BookingManagerService.class); private BookingQueue bookingQueue; private BookingDao bookingDao; diff --git a/cuebot/src/main/java/com/imageworks/spcue/service/DependManagerService.java b/cuebot/src/main/java/com/imageworks/spcue/service/DependManagerService.java index 187e81331..2a82c099d 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/service/DependManagerService.java +++ b/cuebot/src/main/java/com/imageworks/spcue/service/DependManagerService.java @@ -23,7 +23,8 @@ import java.util.List; import java.util.Set; -import org.apache.log4j.Logger; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.LogManager; import org.springframework.dao.DataIntegrityViolationException; import org.springframework.dao.DataRetrievalFailureException; import org.springframework.transaction.annotation.Propagation; @@ -63,7 +64,7 @@ @Transactional public class DependManagerService implements DependManager { - private static final Logger logger = Logger.getLogger(DependManagerService.class); + private static final Logger logger = LogManager.getLogger(DependManagerService.class); private DependDao dependDao; private JobDao jobDao; diff --git a/cuebot/src/main/java/com/imageworks/spcue/service/EmailSupport.java b/cuebot/src/main/java/com/imageworks/spcue/service/EmailSupport.java index ef9c2d32a..b25e7d520 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/service/EmailSupport.java +++ b/cuebot/src/main/java/com/imageworks/spcue/service/EmailSupport.java @@ -33,7 +33,8 @@ import java.util.Map; import java.util.Properties; -import org.apache.log4j.Logger; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.LogManager; import org.apache.velocity.Template; import org.apache.velocity.VelocityContext; import org.apache.velocity.app.VelocityEngine; @@ -65,7 +66,7 @@ public class EmailSupport { private final Map imageMap; - private static final Logger logger = Logger.getLogger(EmailSupport.class); + private static final Logger logger = LogManager.getLogger(EmailSupport.class); public EmailSupport() { diff --git a/cuebot/src/main/java/com/imageworks/spcue/service/FilterManagerService.java b/cuebot/src/main/java/com/imageworks/spcue/service/FilterManagerService.java index 6d4382714..eb49a9c51 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/service/FilterManagerService.java +++ b/cuebot/src/main/java/com/imageworks/spcue/service/FilterManagerService.java @@ -23,7 +23,8 @@ import java.util.List; import java.util.regex.Pattern; -import org.apache.log4j.Logger; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.LogManager; import org.springframework.transaction.annotation.Propagation; import org.springframework.transaction.annotation.Transactional; @@ -62,7 +63,7 @@ @Transactional public class FilterManagerService implements FilterManager { - private static final Logger logger = Logger.getLogger(FilterManagerService.class); + private static final Logger logger = LogManager.getLogger(FilterManagerService.class); private ActionDao actionDao; private MatcherDao matcherDao; diff --git a/cuebot/src/main/java/com/imageworks/spcue/service/HistoricalSupport.java b/cuebot/src/main/java/com/imageworks/spcue/service/HistoricalSupport.java index 0eb6d1213..74c256729 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/service/HistoricalSupport.java +++ b/cuebot/src/main/java/com/imageworks/spcue/service/HistoricalSupport.java @@ -21,12 +21,13 @@ import java.util.List; -import org.apache.log4j.Logger; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.LogManager; import com.imageworks.spcue.JobInterface; public class HistoricalSupport { - private static final Logger logger = Logger.getLogger(HistoricalSupport.class); + private static final Logger logger = LogManager.getLogger(HistoricalSupport.class); private HistoricalManager historicalManager; diff --git a/cuebot/src/main/java/com/imageworks/spcue/service/HostManagerService.java b/cuebot/src/main/java/com/imageworks/spcue/service/HostManagerService.java index ccd355889..a7c5b0729 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/service/HostManagerService.java +++ b/cuebot/src/main/java/com/imageworks/spcue/service/HostManagerService.java @@ -22,7 +22,8 @@ import java.sql.Timestamp; import java.util.List; -import org.apache.log4j.Logger; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.LogManager; import org.springframework.dao.EmptyResultDataAccessException; import org.springframework.transaction.annotation.Propagation; import org.springframework.transaction.annotation.Transactional; @@ -57,7 +58,7 @@ @Transactional public class HostManagerService implements HostManager { - private static final Logger logger = Logger.getLogger(HostManagerService.class); + private static final Logger logger = LogManager.getLogger(HostManagerService.class); private HostDao hostDao; private RqdClient rqdClient; diff --git a/cuebot/src/main/java/com/imageworks/spcue/service/JmsMover.java b/cuebot/src/main/java/com/imageworks/spcue/service/JmsMover.java index bc5be1779..ce231331b 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/service/JmsMover.java +++ b/cuebot/src/main/java/com/imageworks/spcue/service/JmsMover.java @@ -29,7 +29,8 @@ import com.google.gson.Gson; import com.google.gson.GsonBuilder; -import org.apache.log4j.Logger; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.LogManager; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.core.env.Environment; import org.springframework.jms.JmsException; @@ -39,7 +40,7 @@ import com.imageworks.spcue.util.CueExceptionUtil; public class JmsMover extends ThreadPoolExecutor { - private static final Logger logger = Logger.getLogger(JmsMover.class); + private static final Logger logger = LogManager.getLogger(JmsMover.class); private final Gson gson = new GsonBuilder().serializeNulls().create(); @Autowired diff --git a/cuebot/src/main/java/com/imageworks/spcue/service/JobLauncher.java b/cuebot/src/main/java/com/imageworks/spcue/service/JobLauncher.java index 14f2a5741..f46616115 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/service/JobLauncher.java +++ b/cuebot/src/main/java/com/imageworks/spcue/service/JobLauncher.java @@ -23,7 +23,8 @@ import java.util.HashSet; import java.util.Set; -import org.apache.log4j.Logger; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.LogManager; import org.springframework.beans.BeansException; import org.springframework.context.ApplicationContext; import org.springframework.context.ApplicationContextAware; @@ -43,7 +44,7 @@ * Job launching functions. */ public class JobLauncher implements ApplicationContextAware { - private static final Logger logger = Logger.getLogger(JobLauncher.class); + private static final Logger logger = LogManager.getLogger(JobLauncher.class); private ApplicationContext context; private JobManager jobManager; diff --git a/cuebot/src/main/java/com/imageworks/spcue/service/JobManagerService.java b/cuebot/src/main/java/com/imageworks/spcue/service/JobManagerService.java index 6da593712..c1ca1bdfc 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/service/JobManagerService.java +++ b/cuebot/src/main/java/com/imageworks/spcue/service/JobManagerService.java @@ -22,7 +22,8 @@ import java.util.List; import com.google.common.collect.Sets; -import org.apache.log4j.Logger; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.LogManager; import org.springframework.dao.DataAccessException; import org.springframework.transaction.annotation.Propagation; import org.springframework.transaction.annotation.Transactional; @@ -70,7 +71,7 @@ @Transactional public class JobManagerService implements JobManager { - private static final Logger logger = Logger.getLogger(JobManagerService.class); + private static final Logger logger = LogManager.getLogger(JobManagerService.class); private JobDao jobDao; private ShowDao showDao; diff --git a/cuebot/src/main/java/com/imageworks/spcue/service/JobManagerSupport.java b/cuebot/src/main/java/com/imageworks/spcue/service/JobManagerSupport.java index c73c30eb6..b2db74d59 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/service/JobManagerSupport.java +++ b/cuebot/src/main/java/com/imageworks/spcue/service/JobManagerSupport.java @@ -22,7 +22,8 @@ import java.util.Collection; import java.util.List; -import org.apache.log4j.Logger; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.LogManager; import org.springframework.dao.DataAccessException; import org.springframework.dao.EmptyResultDataAccessException; @@ -55,7 +56,7 @@ * A non-transaction support class for managing jobs. */ public class JobManagerSupport { - private static final Logger logger = Logger.getLogger(JobManagerSupport.class); + private static final Logger logger = LogManager.getLogger(JobManagerSupport.class); private JobManager jobManager; private DependManager dependManager; diff --git a/cuebot/src/main/java/com/imageworks/spcue/service/JobSpec.java b/cuebot/src/main/java/com/imageworks/spcue/service/JobSpec.java index 293ce9730..269a9f4af 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/service/JobSpec.java +++ b/cuebot/src/main/java/com/imageworks/spcue/service/JobSpec.java @@ -33,7 +33,8 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; -import org.apache.log4j.Logger; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.LogManager; import org.jdom.Document; import org.jdom.Element; import org.jdom.input.SAXBuilder; @@ -57,7 +58,7 @@ import com.imageworks.spcue.util.CueUtil; public class JobSpec { - private static final Logger logger = Logger.getLogger(JobSpec.class); + private static final Logger logger = LogManager.getLogger(JobSpec.class); private String facility; diff --git a/cuebot/src/main/java/com/imageworks/spcue/service/LocalBookingSupport.java b/cuebot/src/main/java/com/imageworks/spcue/service/LocalBookingSupport.java index a242382bb..7bf7db136 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/service/LocalBookingSupport.java +++ b/cuebot/src/main/java/com/imageworks/spcue/service/LocalBookingSupport.java @@ -19,7 +19,8 @@ package com.imageworks.spcue.service; -import org.apache.log4j.Logger; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.LogManager; import com.imageworks.spcue.DispatchHost; import com.imageworks.spcue.FrameInterface; @@ -37,7 +38,7 @@ */ public class LocalBookingSupport { - private static final Logger logger = Logger.getLogger(LocalBookingSupport.class); + private static final Logger logger = LogManager.getLogger(LocalBookingSupport.class); private HostManager hostManager; private LocalDispatcher localDispatcher; diff --git a/cuebot/src/main/java/com/imageworks/spcue/service/MaintenanceManagerSupport.java b/cuebot/src/main/java/com/imageworks/spcue/service/MaintenanceManagerSupport.java index fdc44eddd..84049bd61 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/service/MaintenanceManagerSupport.java +++ b/cuebot/src/main/java/com/imageworks/spcue/service/MaintenanceManagerSupport.java @@ -21,7 +21,8 @@ import java.util.List; -import org.apache.log4j.Logger; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.LogManager; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.core.env.Environment; import org.springframework.jdbc.CannotGetJdbcConnectionException; @@ -42,7 +43,7 @@ public class MaintenanceManagerSupport { - private static final Logger logger = Logger.getLogger(MaintenanceManagerSupport.class); + private static final Logger logger = LogManager.getLogger(MaintenanceManagerSupport.class); @Autowired private Environment env; diff --git a/cuebot/src/main/java/com/imageworks/spcue/service/RedirectService.java b/cuebot/src/main/java/com/imageworks/spcue/service/RedirectService.java index 23f164c53..51bf4211d 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/service/RedirectService.java +++ b/cuebot/src/main/java/com/imageworks/spcue/service/RedirectService.java @@ -20,7 +20,8 @@ import javax.annotation.Resource; -import org.apache.log4j.Logger; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.LogManager; import org.springframework.dao.CannotSerializeTransactionException; import org.springframework.dao.DuplicateKeyException; import org.springframework.transaction.PlatformTransactionManager; @@ -37,7 +38,7 @@ public class RedirectService { private static final Logger logger = - Logger.getLogger(RedirectService.class); + LogManager.getLogger(RedirectService.class); @Resource private PlatformTransactionManager txManager; diff --git a/cuebot/src/main/java/com/imageworks/spcue/service/WhiteboardService.java b/cuebot/src/main/java/com/imageworks/spcue/service/WhiteboardService.java index 694a58643..5fe08e1f6 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/service/WhiteboardService.java +++ b/cuebot/src/main/java/com/imageworks/spcue/service/WhiteboardService.java @@ -21,7 +21,8 @@ import java.util.List; -import org.apache.log4j.Logger; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.LogManager; import org.springframework.transaction.annotation.Propagation; import org.springframework.transaction.annotation.Transactional; @@ -111,7 +112,7 @@ public class WhiteboardService implements Whiteboard { @SuppressWarnings("unused") - private static final Logger logger = Logger.getLogger(WhiteboardService.class); + private static final Logger logger = LogManager.getLogger(WhiteboardService.class); private WhiteboardDao whiteboardDao; diff --git a/cuebot/src/main/java/com/imageworks/spcue/servlet/JobLaunchServlet.java b/cuebot/src/main/java/com/imageworks/spcue/servlet/JobLaunchServlet.java index 284b6739a..76040b7bd 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/servlet/JobLaunchServlet.java +++ b/cuebot/src/main/java/com/imageworks/spcue/servlet/JobLaunchServlet.java @@ -25,7 +25,8 @@ import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; -import org.apache.log4j.Logger; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.LogManager; import org.springframework.web.servlet.FrameworkServlet; import com.imageworks.spcue.BuildableJob; @@ -39,7 +40,7 @@ @SuppressWarnings("serial") public class JobLaunchServlet extends FrameworkServlet { - private static final Logger logger = Logger.getLogger(JobLaunchServlet.class); + private static final Logger logger = LogManager.getLogger(JobLaunchServlet.class); private JobLauncher jobLauncher; diff --git a/cuebot/src/main/java/com/imageworks/spcue/util/CueExceptionUtil.java b/cuebot/src/main/java/com/imageworks/spcue/util/CueExceptionUtil.java index e947815a2..3879914b1 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/util/CueExceptionUtil.java +++ b/cuebot/src/main/java/com/imageworks/spcue/util/CueExceptionUtil.java @@ -23,7 +23,8 @@ import java.io.StringWriter; import java.io.Writer; -import org.apache.log4j.Logger; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.LogManager; /** * Utility class for handling and logging exceptions @@ -52,7 +53,7 @@ public static String getStackTrace(Throwable aThrowable) { * @return String */ public static void logStackTrace(String msg, Throwable aThrowable) { - Logger error_logger = Logger.getLogger(CueExceptionUtil.class); + Logger error_logger = LogManager.getLogger(CueExceptionUtil.class); error_logger.info("Caught unexpected exception caused by: " + aThrowable); error_logger.info("StackTrace: \n" + getStackTrace(aThrowable)); if (aThrowable.getCause() != null) { diff --git a/cuebot/src/main/java/com/imageworks/spcue/util/CueUtil.java b/cuebot/src/main/java/com/imageworks/spcue/util/CueUtil.java index 2a7438f49..88b325483 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/util/CueUtil.java +++ b/cuebot/src/main/java/com/imageworks/spcue/util/CueUtil.java @@ -46,7 +46,8 @@ import javax.mail.internet.MimeMultipart; import javax.mail.util.ByteArrayDataSource; -import org.apache.log4j.Logger; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.LogManager; import org.springframework.core.env.Environment; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; @@ -62,7 +63,7 @@ @Component public final class CueUtil { - private static final Logger logger = Logger.getLogger(CueUtil.class); + private static final Logger logger = LogManager.getLogger(CueUtil.class); private static String smtpHost = ""; @Autowired private Environment env; diff --git a/cuebot/src/main/resources/log4j.properties b/cuebot/src/main/resources/log4j.properties deleted file mode 100644 index 5f44dbee9..000000000 --- a/cuebot/src/main/resources/log4j.properties +++ /dev/null @@ -1,51 +0,0 @@ -############################################################## -# SpCue Logging Configuration -############################################################## - -############################################################### -# Root Logger -# Logs Application wide INFO messages / Tomcat messges -############################################################### - -log4j.rootLogger=INFO, STDOUT, FILE -log4j.appender.STDOUT=org.apache.log4j.ConsoleAppender -log4j.appender.STDOUT.Threshold=WARN -log4j.appender.STDOUT.layout=org.apache.log4j.PatternLayout -log4j.appender.STDOUT.layout.ConversionPattern=%d %p %t %c - %m%n -log4j.appender.FILE=org.apache.log4j.RollingFileAppender -log4j.appender.FILE.File=logs/spcue.log -log4j.appender.FILE.MaxFileSize=10MB -log4j.appender.FILE.MaxBackupIndex=10 -log4j.appender.FILE.layout=org.apache.log4j.PatternLayout -log4j.appender.FILE.layout.ConversionPattern=%d %p %t %c - %m%n - -log4j.category.API=INFO, API -log4j.additivity.API=false -log4j.appender.API=org.apache.log4j.RollingFileAppender -log4j.appender.API.File=logs/api.log -log4j.appender.API.MaxFileSize=10MB -log4j.appender.API.MaxBackupIndex=20 -log4j.appender.API.layout=org.apache.log4j.PatternLayout -log4j.appender.API.layout.ConversionPattern=%d:%m%n - -log4j.category.HEALTH=DEBUG, HEALTH -log4j.additivity.HEALTH=false -log4j.appender.HEALTH=org.apache.log4j.RollingFileAppender -log4j.appender.HEALTH.File=logs/health.log -log4j.appender.HEALTH.MaxFileSize=10MB -log4j.appender.HEALTH.MaxBackupIndex=20 -log4j.appender.HEALTH.layout=org.apache.log4j.PatternLayout -log4j.appender.HEALTH.layout.ConversionPattern=%d:%m%n - -log4j.logger.org.apache.catalina=INFO -log4j.logger.com.imageworks.spcue=DEBUG -log4j.logger.com.imageworks.spcue.dispatcher.RqdReportManagerService=DEBUG -log4j.logger.com.imageworks.spcue.service.HostManagerService=TRACE -log4j.logger.com.imageworks.spcue.dispatcher=TRACE - -#log4j.logger.org.springframework=DEBUG - -# Very verbose sql output: -#log4j.logger.org.springframework.jdbc.core=DEBUG -#log4j.logger.org.springframework.jdbc.core.JdbcTemplate=DEBUG -#log4j.logger.org.springframework.jdbc.core.StatementCreatorUtils=TRACE diff --git a/cuebot/src/main/resources/log4j2.properties b/cuebot/src/main/resources/log4j2.properties new file mode 100644 index 000000000..b924a762f --- /dev/null +++ b/cuebot/src/main/resources/log4j2.properties @@ -0,0 +1,103 @@ +############################################################## +# OpenCue Logging Configuration +############################################################## + +# Log4j uses "appenders" and "loggers". Loggers define the logging behavior within the +# application. Appenders deliver the log messages to the intended targets. Loggers must +# be associated with appenders in order for log messages to be written out. + +# Stdout. +appender.console.type = Console +appender.console.name = STDOUT +appender.console.layout.type = PatternLayout +appender.console.layout.pattern = %d %p %t %c - %m%n +appender.console.filter.threshold.type = ThresholdFilter +appender.console.filter.threshold.level = warn + +# Main log file. +appender.rolling.type = RollingFile +appender.rolling.name = FILE +appender.rolling.fileName = logs/spcue.log +appender.rolling.filePattern = logs/spcue.log.%i +appender.rolling.layout.type = PatternLayout +appender.rolling.layout.pattern = %d %p %t %c - %m%n +appender.rolling.policies.type = Policies +appender.rolling.policies.size.type = SizeBasedTriggeringPolicy +appender.rolling.policies.size.size=10MB +appender.rolling.strategy.type = DefaultRolloverStrategy +appender.rolling.strategy.max = 10 + +# API log file, for logging API requests only. +appender.api.type = RollingFile +appender.api.name = API +appender.api.fileName = logs/api.log +appender.api.filePattern = logs/api.log.%i +appender.api.layout.type = PatternLayout +appender.api.layout.pattern = %d:%m%n +appender.api.policies.type = Policies +appender.api.policies.size.type = SizeBasedTriggeringPolicy +appender.api.policies.size.size = 10MB +appender.api.strategy.type = DefaultRolloverStrategy +appender.api.strategy.max = 20 +appender.api.filter.threshold.type = ThresholdFilter +appender.api.filter.threshold.level = info + +# HEALTH log file +appender.health.type = RollingFile +appender.health.name = HEALTH +appender.health.fileName = logs/health.log +appender.health.filePattern = logs/health.log.%i +appender.health.layout.type = PatternLayout +appender.health.layout.pattern = %d:%m%n +appender.health.policies.type = Policies +appender.health.policies.size.type = SizeBasedTriggeringPolicy +appender.health.policies.size.size = 10MB +appender.health.strategy.type = DefaultRolloverStrategy +appender.health.strategy.max = 20 +appender.health.filter.threshold.type = ThresholdFilter +appender.health.filter.threshold.level = debug + +# Root-level logger. All messages will go to both stdout and the main log file, though they +# may not appear there if the appender filters based on log level. For example INFO messages +# will not appear by default in stdout as the default log level for that appender is WARN. +rootLogger.level = info +rootLogger.appenderRef.stdout.ref = STDOUT +rootLogger.appenderRef.file.ref = FILE + +# API logger. Does not inherit from the root logger, so only API requests will be logged. +logger.api.name = API +logger.api.level = info +logger.api.additivity = false +logger.api.appenderRef.api.ref = API + +# HEALTH logger. Does not inherit from the root logger, so only HEALTH requests will be logged. +logger.health.name = HEALTH +logger.health.level = debug +logger.health.additivity = false +logger.health.appenderRef.health.ref = HEALTH + +# Child loggers. These inherit from the root logger, so will be sent to the relevant appenders. +# This allows us to increase verbosity for specific modules. + +logger.catalina.name = org.apache.catalina +logger.catalina.level = info + +logger.spcue.name = com.imageworks.spcue +logger.spcue.level = debug + +logger.rqdReport.name = com.imageworks.spcue.dispatcher.RqdReportManagerService +logger.rqdReport.level = debug + +logger.hostManager.name = com.imageworks.spcue.service.HostManagerService +logger.hostManager.level = trace + +logger.dispatcher.name = com.imageworks.spcue.dispatcher +logger.dispatcher.level = trace + +# For very verbose sql output: +# logger.sql.name = org.springframework.jdbc.core +# logger.sql.level = debug +# logger.sqlJdbcTemplate.name = org.springframework.jdbc.core.JdbcTemplate +# logger.sqlJdbcTemplate.level = debug +# logger.sqlStatementCreator.name = org.springframework.jdbc.core.StatementCreatorUtils +# logger.sqlStatementCreator.level = trace From 939811a87ac83b8f6985af07216a38219375c370 Mon Sep 17 00:00:00 2001 From: Brian Cipriano Date: Fri, 16 Dec 2022 15:04:00 -0500 Subject: [PATCH 02/56] Notes from Dec 7 TSC meeting. (#1231) --- tsc/meetings/2022-12-07.md | 51 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) create mode 100644 tsc/meetings/2022-12-07.md diff --git a/tsc/meetings/2022-12-07.md b/tsc/meetings/2022-12-07.md new file mode 100644 index 000000000..6fc9286cf --- /dev/null +++ b/tsc/meetings/2022-12-07.md @@ -0,0 +1,51 @@ +# OpenCue TSC Meeting Notes 7 Dec 2022 + +Secretary: Brian Cipriano + +Agenda/Notes: + +* Postgres upgrade/fixes + * https://github.com/AcademySoftwareFoundation/OpenCue/pull/1199 + * PR updated with some new research. + * DBA contact has reproduced problem but doesn't support Docker deployments. Confirmed this is + embedded-postgres issue only. + * Filed a ticket with the upstream project. Got a few suggestions. + * Will be hard to verify performance issues until it's in production and hard to roll back. + * Conclusion: Mac+Docker for production deployments is uncommon, this is mostly for developers. + Let's work around this for now, and look into the suggestions from the upstream ticket. + Hopefully we can track down the problem and avoid having to merge this PR. +* Log4j update + * https://github.com/AcademySoftwareFoundation/OpenCue/pull/1080 + * Older PR, requested by user who tested and verified. + * Confirmed, we are good to merge this now. +* PySide6 + * https://github.com/AcademySoftwareFoundation/OpenCue/pull/1195 + * SPI: difficult to production test, PySide6 does not fit nicely into packaging system. + * Conclusion: we'll add the compatibility layer now, test as best we can, and merge as an + experimental feature. Issues may still be present but we can fix these as we go, and it will + be a better situation than we currently have, where CueGUI is not available at all for users + who can't access PySide2. +* Postgres query issues + * New issue with very slow queries on the database side. + * Upgrade happened 3-4 months ago but symptoms didn't present until heavy production load. + * Debugged issue, found culprit is the `show` table, particularly a few stats columns. These + columns are updated multiple times per second under heavy load, and many other critical + queries join to the show table. This slows down the whole system. + * PR coming soon to separate these columns out into their own table/view. +* Removing dead code + * https://github.com/AcademySoftwareFoundation/OpenCue/pull/1178/files + * Can we ignore the version bump here? + * Diego will look into it offline. +* akim-ruslanov PRs need update + * https://github.com/AcademySoftwareFoundation/OpenCue/pull/1168 + * https://github.com/AcademySoftwareFoundation/OpenCue/pull/1167 + * https://github.com/AcademySoftwareFoundation/OpenCue/pull/1165 + * Diego will look into these offline. +* Blender update + * Blender has made progress on improved Python bindings. Should make it easier to avoid + compatibility issues in the future. + * They will support a build using VFX reference platform versions. + * March release for the initial implementation, this will be improved over the next few cycles. + * JT looking to hand this off soon. + * Nuwan is interested in working on the plugin as well. He should proceed, and we'll sync his + and JT's work if needed, or maybe JT will just use it as the base for his own work. From 133687d6328bcf42b8c350b8fbdae6605d98e43d Mon Sep 17 00:00:00 2001 From: Brian Cipriano Date: Mon, 9 Jan 2023 15:19:28 -0500 Subject: [PATCH 03/56] Remove obsolete CREDITS file. (#1234) --- CREDITS | 37 ------------------------------------- 1 file changed, 37 deletions(-) delete mode 100644 CREDITS diff --git a/CREDITS b/CREDITS deleted file mode 100644 index 988a755e9..000000000 --- a/CREDITS +++ /dev/null @@ -1,37 +0,0 @@ -cue3bot - 3720 Matt Chambers - 167 John Welborn - 123 Michael Zhang - 50 J Robert Ray - 27 Bond-Jay Ting - 22 Kasra Faghihi - 14 Blair Zajac - 2 Kevin Coats - 1 Jordon Phillips - -rqd - 473 John Welborn - 25 J Robert Ray - 6 Yudi Xue - 2 Blair Zajac - 2 Michael Zhang - 2 Jordon Phillips - 1 Kasra Faghihi - -spi_cue - 252 John Welborn - 226 Matt Chambers - 12 Jordon Phillips - 8 Yudi Xue - 5 J Robert Ray - 4 Michael Zhang - 3 Blair Zajac - -python_ice_server - 27 Blair Zajac - 8 Cottalango Leon - 5 J Robert Ray - 3 John Welborn - 2 Michael Zhang - 1 Geo Snelling - 1 Sam Richards From a4a9cd1da02ed1de6aeeb42af8d6553378c1e104 Mon Sep 17 00:00:00 2001 From: Olivier Evers Date: Tue, 10 Jan 2023 21:59:16 +0100 Subject: [PATCH 04/56] [rqd] Add some missing env vars on Windows. (#1225) --- rqd/rqd/rqcore.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/rqd/rqd/rqcore.py b/rqd/rqd/rqcore.py index 646c4f73d..224485f2b 100644 --- a/rqd/rqd/rqcore.py +++ b/rqd/rqd/rqcore.py @@ -101,8 +101,9 @@ def __createEnvVariables(self): self.frameEnv["MAIL"] = "/usr/mail/%s" % self.runFrame.user_name self.frameEnv["HOME"] = "/net/homedirs/%s" % self.runFrame.user_name elif platform.system() == "Windows": - self.frameEnv["APPDATA"] = os.environ["APPDATA"] - self.frameEnv["SYSTEMROOT"] = os.environ["SYSTEMROOT"] + for variable in ["SYSTEMROOT", "APPDATA", "TMP", "COMMONPROGRAMFILES"]: + if variable in os.environ: + self.frameEnv[variable] = os.environ[variable] for key, value in self.runFrame.environment.items(): if key == 'PATH': From 58fcb4a1472e717b14175231bcb5241b2395958f Mon Sep 17 00:00:00 2001 From: romainf-ubi <117918548+romainf-ubi@users.noreply.github.com> Date: Thu, 12 Jan 2023 12:15:14 -0500 Subject: [PATCH 05/56] [cuebot] Fix malformed SQL queries. (#1222) --- .../spcue/dao/postgres/DispatchQuery.java | 14 +++++++------- .../spcue/dao/postgres/FilterDaoJdbc.java | 4 ++-- .../imageworks/spcue/dao/postgres/JobDaoJdbc.java | 4 ++-- .../imageworks/spcue/dao/postgres/ShowDaoJdbc.java | 2 +- .../spcue/test/dao/postgres/LayerDaoTests.java | 4 ++-- 5 files changed, 14 insertions(+), 14 deletions(-) diff --git a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/DispatchQuery.java b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/DispatchQuery.java index 865472d1a..1d1f7210f 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/DispatchQuery.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/DispatchQuery.java @@ -230,7 +230,7 @@ private static final String replaceQueryForFifo(String query) { "AND " + "job.pk_facility = ? " + "AND " + - "(job.str_os = ? OR job.str_os IS NULL)" + + "(job.str_os = ? OR job.str_os IS NULL) " + "AND " + "job.pk_job IN ( " + "SELECT " + @@ -256,7 +256,7 @@ private static final String replaceQueryForFifo(String query) { "AND " + "j.pk_facility = ? " + "AND " + - "(j.str_os = ? OR j.str_os IS NULL)" + + "(j.str_os = ? OR j.str_os IS NULL) " + "AND " + "(CASE WHEN lst.int_waiting_count > 0 THEN lst.pk_layer ELSE NULL END) = l.pk_layer " + "AND " + @@ -333,7 +333,7 @@ private static final String replaceQueryForFifo(String query) { "AND " + "job.pk_facility = ? " + "AND " + - "(job.str_os = ? OR job.str_os IS NULL)" + + "(job.str_os = ? OR job.str_os IS NULL) " + "AND " + "job.pk_job IN ( " + "SELECT /* index (h i_str_host_tag) */ " + @@ -354,7 +354,7 @@ private static final String replaceQueryForFifo(String query) { "AND " + "j.pk_facility = ? " + "AND " + - "(j.str_os = ? OR j.str_os IS NULL)" + + "(j.str_os = ? OR j.str_os IS NULL) " + "AND " + "(CASE WHEN lst.int_waiting_count > 0 THEN lst.pk_layer ELSE NULL END) = l.pk_layer " + "AND " + @@ -426,7 +426,7 @@ private static final String replaceQueryForFifo(String query) { "AND " + "(folder_resource.int_max_gpus = -1 OR folder_resource.int_gpus < folder_resource.int_max_gpus) " + "AND " + - "job_resource.int_priority > ?" + + "job_resource.int_priority > ? " + "AND " + "job_resource.int_cores < job_resource.int_max_cores " + "AND " + @@ -438,7 +438,7 @@ private static final String replaceQueryForFifo(String query) { "AND " + "job.pk_facility = ? " + "AND " + - "(job.str_os = ? OR job.str_os IS NULL)" + + "(job.str_os = ? OR job.str_os IS NULL) " + "AND " + "job.pk_job IN ( " + "SELECT /* index (h i_str_host_tag) */ " + @@ -457,7 +457,7 @@ private static final String replaceQueryForFifo(String query) { "AND " + "j.pk_facility = ? " + "AND " + - "(j.str_os = ? OR j.str_os IS NULL)" + + "(j.str_os = ? OR j.str_os IS NULL) " + "AND " + "(CASE WHEN lst.int_waiting_count > 0 THEN lst.pk_layer ELSE NULL END) = l.pk_layer " + "AND " + diff --git a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/FilterDaoJdbc.java b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/FilterDaoJdbc.java index a1cb7d2cf..ee30d5f8e 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/FilterDaoJdbc.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/FilterDaoJdbc.java @@ -54,7 +54,7 @@ public class FilterDaoJdbc extends JdbcDaoSupport implements FilterDao { private static final String GET_ACTIVE_FILTERS = "SELECT " + - "filter.*" + + "filter.* " + "FROM " + "filter " + "WHERE " + @@ -66,7 +66,7 @@ public class FilterDaoJdbc extends JdbcDaoSupport implements FilterDao { private static final String GET_FILTERS = "SELECT " + - "filter.*" + + "filter.* " + "FROM " + "filter " + "WHERE " + diff --git a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/JobDaoJdbc.java b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/JobDaoJdbc.java index 890f5ea81..d32a5a259 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/JobDaoJdbc.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/JobDaoJdbc.java @@ -437,7 +437,7 @@ public void updateMaxRSS(JobInterface job, long value) { "str_visible_name = NULL, " + "ts_stopped = current_timestamp "+ "WHERE " + - "str_state = 'PENDING'" + + "str_state = 'PENDING' " + "AND " + "pk_job = ?"; @@ -945,7 +945,7 @@ public void updateParent(JobInterface job, GroupDetail dest, Inherit[] inherits) "AND " + "job.b_auto_book = true " + "AND " + - "job_stat.int_waiting_count != 0" + + "job_stat.int_waiting_count != 0 " + "AND " + "job_resource.int_cores < job_resource.int_max_cores " + "AND " + diff --git a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/ShowDaoJdbc.java b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/ShowDaoJdbc.java index 5d674ed82..8361a6c5f 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/ShowDaoJdbc.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/ShowDaoJdbc.java @@ -147,7 +147,7 @@ public void insertShow(ShowEntity show) { "SELECT " + "COUNT(show.pk_show) " + "FROM " + - "show LEFT JOIN show_alias ON (show.pk_show = show_alias.pk_show )" + + "show LEFT JOIN show_alias ON (show.pk_show = show_alias.pk_show) " + "WHERE " + "(show.str_name = ? OR show_alias.str_name = ?) "; public boolean showExists(String name) { diff --git a/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/LayerDaoTests.java b/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/LayerDaoTests.java index 8ad2f5bac..83c8a466b 100644 --- a/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/LayerDaoTests.java +++ b/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/LayerDaoTests.java @@ -522,7 +522,7 @@ public void isOptimizable() { layer.getLayerId()); jdbcTemplate.update( - "UPDATE layer_usage SET int_core_time_success = 3600 * 6" + + "UPDATE layer_usage SET int_core_time_success = 3600 * 6 " + "WHERE pk_layer=?", layer.getLayerId()); assertFalse(layerDao.isOptimizable(layer, 5, 3600)); @@ -532,7 +532,7 @@ public void isOptimizable() { * Assert True */ jdbcTemplate.update( - "UPDATE layer_usage SET int_core_time_success = 3500 * 5" + + "UPDATE layer_usage SET int_core_time_success = 3500 * 5 " + "WHERE pk_layer=?", layer.getLayerId()); assertTrue(layerDao.isOptimizable(layer, 5, 3600)); From 3b2326ebbb3d1f4395cdcb66d0b41a556ee57e17 Mon Sep 17 00:00:00 2001 From: romainf-ubi <117918548+romainf-ubi@users.noreply.github.com> Date: Thu, 12 Jan 2023 12:30:15 -0500 Subject: [PATCH 06/56] [rqd] Add pynput to requirements.txt. (#1230) --- requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements.txt b/requirements.txt index f465c5910..92b375d60 100644 --- a/requirements.txt +++ b/requirements.txt @@ -13,5 +13,6 @@ protobuf==3.17.3;python_version<"3.0" psutil==5.6.7 pyfakefs==3.6 pylint==2.6.0;python_version>="3.7" +pynput==1.7.6 PyYAML==5.1 six==1.11.0 From 1a920dcb7b79668777388c69c4806e2e6d29e69b Mon Sep 17 00:00:00 2001 From: romainf-ubi <117918548+romainf-ubi@users.noreply.github.com> Date: Thu, 12 Jan 2023 13:09:14 -0500 Subject: [PATCH 07/56] Upgrade PySide2 to 5.15.2.1. (#1226) --- requirements_gui.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements_gui.txt b/requirements_gui.txt index d9aa53b35..c589c3517 100644 --- a/requirements_gui.txt +++ b/requirements_gui.txt @@ -1 +1 @@ -PySide2==5.15.2 +PySide2==5.15.2.1 From 27bc9e16713d437b92cee87a7c148c77ddaf2cec Mon Sep 17 00:00:00 2001 From: Brian Cipriano Date: Mon, 16 Jan 2023 14:25:08 -0500 Subject: [PATCH 08/56] [sandbox] Stability improvements. (#1244) --- docker-compose.yml | 2 +- sandbox/flyway.Dockerfile | 6 ++---- sandbox/get-latest-release-tag.sh | 12 ++++++++++++ sandbox/migrate.sh | 3 ++- 4 files changed, 17 insertions(+), 6 deletions(-) create mode 100755 sandbox/get-latest-release-tag.sh diff --git a/docker-compose.yml b/docker-compose.yml index 554735581..656dec4bc 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -2,7 +2,7 @@ version: '3' services: db: - image: postgres + image: postgres:15.1 environment: - POSTGRES_USER=cuebot - POSTGRES_PASSWORD=cuebot_password diff --git a/sandbox/flyway.Dockerfile b/sandbox/flyway.Dockerfile index d3d45b998..086d47bbd 100644 --- a/sandbox/flyway.Dockerfile +++ b/sandbox/flyway.Dockerfile @@ -1,10 +1,8 @@ -FROM centos +FROM almalinux:8.7 -ARG FLYWAY_VERSION=8.5.4 +ARG FLYWAY_VERSION=9.9.0 # Get flyway -RUN sed -i 's/mirrorlist/#mirrorlist/g' /etc/yum.repos.d/CentOS-* -RUN sed -i 's|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g' /etc/yum.repos.d/CentOS-* RUN yum install -y tar java-1.8.0-openjdk postgresql-jdbc nc postgresql RUN curl -O https://repo1.maven.org/maven2/org/flywaydb/flyway-commandline/${FLYWAY_VERSION}/flyway-commandline-${FLYWAY_VERSION}-linux-x64.tar.gz RUN tar -xzf flyway-commandline-${FLYWAY_VERSION}-linux-x64.tar.gz diff --git a/sandbox/get-latest-release-tag.sh b/sandbox/get-latest-release-tag.sh new file mode 100755 index 000000000..0aa552efa --- /dev/null +++ b/sandbox/get-latest-release-tag.sh @@ -0,0 +1,12 @@ +#!/bin/bash + +# Script for fetching the latest release version of OpenCue. +# - `curl` fetches all of the metadata for the latest release, in JSON format. +# - `grep` filters for just the `"tag_name": "v1.2.3"` line. +# - `cut` extracts the `v1.2.3` value from the `tag_name` line. +# - `tr` removes the `v` to leave us with the final version number e.g. `1.2.3`. + +curl -s https://api.github.com/repos/AcademySoftwareFoundation/OpenCue/releases/latest \ + | grep tag_name \ + | cut -d \" -f 4 \ + | tr -d v diff --git a/sandbox/migrate.sh b/sandbox/migrate.sh index 529dca834..b7af1cdfd 100755 --- a/sandbox/migrate.sh +++ b/sandbox/migrate.sh @@ -1,5 +1,6 @@ #!/bin/bash +set -e until nc --send-only $PGHOST $PGPORT < /dev/null do @@ -13,7 +14,7 @@ do sleep 2 done -# Apply the flyway database migrations. +echo "Applying database migrations..." ./flyway migrate -user=${PGUSER} -password=${PGPASSWORD} -url="jdbc:postgresql://${PGHOST}:${PGPORT}/${PGDATABASE}" -locations='filesystem:/opt/migrations' # Check if a show exists, if not apply demo data From 14dddd36f16348c9c6778435136c7904302a0ad9 Mon Sep 17 00:00:00 2001 From: Brian Cipriano Date: Fri, 20 Jan 2023 13:00:14 -0500 Subject: [PATCH 09/56] Notes from Jan 18 TSC meeting. (#1247) --- tsc/meetings/2023-01-18.md | 67 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 67 insertions(+) create mode 100644 tsc/meetings/2023-01-18.md diff --git a/tsc/meetings/2023-01-18.md b/tsc/meetings/2023-01-18.md new file mode 100644 index 000000000..1f34a4726 --- /dev/null +++ b/tsc/meetings/2023-01-18.md @@ -0,0 +1,67 @@ +# OpenCue TSC Meeting Notes 18 Jan 2023 + +Secretary: Brian Cipriano + +Agenda/Notes: + +* New release! + * CI / dev environment issues mostly resolved now, release unblocked. + * Getting back to our monthly release cycle. +* Postgres upgrade + * https://github.com/AcademySoftwareFoundation/OpenCue/pull/1199 + * Upstream: https://github.com/zonkyio/embedded-postgres/issues/99 + * Cuebot Docker image currently does not build on M1 Mac due to the old embedded-postgres + library. + * Newer embedded-postgres binaries produce weird results that fail tests. M1+Docker only. + * [Breakthrough!](https://github.com/zonkyio/embedded-postgres/issues/99#issuecomment-1378159242) + The issue was coming from an old Ubuntu version used for the embedded-postgres build. + * Brian sent https://github.com/zonkyio/embedded-postgres-binaries/pull/64 upstream with a + proposed fix, which upgrades their build process to use a newer Ubuntu version. + * Waiting for review, then waiting for new binaries to be published. But we are able to build + embedded-postgres locally now, and modify Cuebot to use those binaries rather than pull from + Maven. +* PySide6 + * New proposed PR: https://github.com/AcademySoftwareFoundation/OpenCue/pull/1238 + * Use QtPy library as the compatibility layer, works in PySide2 and 6. + * Would like to get more testing in a PySide2 environment. + * This breaks the dependency chain slightly, as the code would now depend on QtPy but not + PySide. However we can specify whatever other dependencies we want in + setup.py/requirements.txt. + * Proposal: master branch will continue to specify PySide 2 as a dependency. Packaging/release + pipelines will also create a PySide 6 version. + * Idea: setup.py is a Python script, so it could use custom logic to specify the PySide6 + dependency if on an M1 mac. setup.py is executed not just at build/packaging time but at + install time as well. + * Will test more using Pyside2 before merging. +* Migrate stats columns + * https://github.com/AcademySoftwareFoundation/OpenCue/pull/1228 + * Test failures to be resolved. + * Diego will take a look. + * Maybe related to some intermittent CI failures we've seen recently, those should be mostly + resolved now. +* CueGUI new config file, cuegui.yaml + * https://github.com/AcademySoftwareFoundation/OpenCue/pull/1242 + * Moves most constants to be editable via an external YAML file. No longer any need to edit + code, and exposes many useful settings to users. + * YAML file can be specified via env var or standard config directories. Sysadmins can + distribute their own cuegui.yaml to all local users. + * Need docs update that includes config guides for all components, including this update. + * This should be the last client-side piece needing a config update. We can now move on to the + main PyPI work. +* Integration tests + * https://github.com/AcademySoftwareFoundation/OpenCue/pull/1245 + * Initial version out for review. Stands up Docker compose environment, tests database, RQD + registration, API, cueadmin. + * Next will need to add launching a job and verifying. +* Batch of GUI bug fixes coming soon. +* RQD systemd changes + * Previously using init.d, now migrating to systemd. + * OOM manager was sometimes killing the parent RQD process rather than the job itself. This + would take the RQD host offline and it would not report on failure cause. Cuebot would then + distribute the culprit job to other hosts, and the problem could proliferate. + * systemd has a feature to help reduce likelihood of this happening. + * Once that's done, it would be good to publish rpm packages as part of packaging/release. + Cuebot does this already, this would standardize among the server-side components. + * RQD pip package may need to include initd/systemd scripts, or docs to help register RQD with + the system, i.e. start on host boot. + * Sysadmins also seem to prefer rpms to pip install. From ad28df3f8b0f7975db15f0c72d8b90e8f742b555 Mon Sep 17 00:00:00 2001 From: Brian Cipriano Date: Fri, 20 Jan 2023 14:13:12 -0500 Subject: [PATCH 10/56] Add wrapper script for cuegui tests. (#1236) --- ci/run_gui_test.sh | 27 +++++++++++++++++++++++++++ ci/run_python_tests.sh | 2 +- cuegui/tests/MenuActions_tests.py | 2 +- 3 files changed, 29 insertions(+), 2 deletions(-) create mode 100755 ci/run_gui_test.sh diff --git a/ci/run_gui_test.sh b/ci/run_gui_test.sh new file mode 100755 index 000000000..b80505bdb --- /dev/null +++ b/ci/run_gui_test.sh @@ -0,0 +1,27 @@ +#!/bin/bash + +# Wrapper script for CueGUI tests. +# +# xvfb-run sometimes crashes on exit, we haven't been able to figure out why yet. +# This means that tests may pass but the xvfb-run crash will generate a non-zero exit code +# and cause our CI pipeline to fail. +# +# We work around this by capturing unit test output and looking for the text that indicates +# tests have passed: +# +# > Ran 209 tests in 4.394s +# > +# > OK +# + +test_log="/tmp/cuegui_result.log" +PYTHONPATH=pycue xvfb-run -d python cuegui/setup.py test | tee ${test_log} + +grep -Pz 'Ran \d+ tests in [0-9\.]+s\n\nOK' ${test_log} +if [ $? -eq 0 ]; then + echo "Detected passing tests" + exit 0 +fi + +echo "Detected test failure" +exit 1 diff --git a/ci/run_python_tests.sh b/ci/run_python_tests.sh index 093ab9c62..c37b539ad 100755 --- a/ci/run_python_tests.sh +++ b/ci/run_python_tests.sh @@ -24,5 +24,5 @@ python rqd/setup.py test # Xvfb no longer supports Python 2. if [[ "$python_version" =~ "Python 3" ]]; then - PYTHONPATH=pycue xvfb-run -d python cuegui/setup.py test + ci/run_gui_test.sh fi diff --git a/cuegui/tests/MenuActions_tests.py b/cuegui/tests/MenuActions_tests.py index 25033c79b..0127cc7ce 100644 --- a/cuegui/tests/MenuActions_tests.py +++ b/cuegui/tests/MenuActions_tests.py @@ -65,7 +65,7 @@ def setUp(self): self.job_actions = cuegui.MenuActions.JobActions(self.widgetMock, mock.Mock(), None, None) def test_jobs(self): - print(cuegui.MenuActions.MenuActions(self.widgetMock, None, None, None).jobs()) + cuegui.MenuActions.MenuActions(self.widgetMock, None, None, None).jobs() def test_unmonitor(self): self.job_actions.unmonitor() From 64a4dfd29827f3797fa458d4ad54d31394f9ee7b Mon Sep 17 00:00:00 2001 From: Brian Cipriano Date: Fri, 20 Jan 2023 14:14:04 -0500 Subject: [PATCH 11/56] Replace deprecated CI actions. (#1239) --- .github/workflows/packaging-pipeline.yml | 20 ++++++++----- .github/workflows/release-pipeline.yml | 29 +++++++++++------- .github/workflows/sonar-cloud-pipeline.yml | 4 +-- .github/workflows/testing-pipeline.yml | 34 +++++++++++----------- 4 files changed, 49 insertions(+), 38 deletions(-) diff --git a/.github/workflows/packaging-pipeline.yml b/.github/workflows/packaging-pipeline.yml index 920d92ca7..6fcfa5676 100644 --- a/.github/workflows/packaging-pipeline.yml +++ b/.github/workflows/packaging-pipeline.yml @@ -43,7 +43,7 @@ jobs: name: Build ${{ matrix.NAME }} steps: - name: Checkout - uses: actions/checkout@v2 + uses: actions/checkout@v3 with: # Fetch all Git history, otherwise the current version number will # not be correctly calculated. @@ -58,6 +58,12 @@ jobs: role-to-assume: ${{ secrets.AWS_S3_ROLE }} role-duration-seconds: 1800 + - name: Login to Docker Hub + uses: docker/login-action@v2 + with: + username: ${{ secrets.DOCKER_USER }} + password: ${{ secrets.DOCKER_PASS }} + - name: Set build ID run: | set -e @@ -66,13 +72,11 @@ jobs: echo "BUILD_ID=$(cat ./VERSION)" >> ${GITHUB_ENV} - name: Build Docker image - uses: docker/build-push-action@v1 + uses: docker/build-push-action@v3 with: - username: ${{ secrets.DOCKER_USER }} - password: ${{ secrets.DOCKER_PASS }} - dockerfile: ${{ matrix.component }}/Dockerfile - repository: opencuebuild/${{ matrix.component }} - tags: ${{ env.BUILD_ID }} + file: ${{ matrix.component }}/Dockerfile + tags: opencuebuild/${{ matrix.component }}:${{ env.BUILD_ID }} + push: true - name: Extract Artifacts run: | @@ -101,7 +105,7 @@ jobs: name: Create Other Build Artifacts steps: - name: Checkout - uses: actions/checkout@v2 + uses: actions/checkout@v3 with: # Fetch all Git history, otherwise the current version number will # not be correctly calculated. diff --git a/.github/workflows/release-pipeline.yml b/.github/workflows/release-pipeline.yml index d87637d61..1111bec54 100644 --- a/.github/workflows/release-pipeline.yml +++ b/.github/workflows/release-pipeline.yml @@ -12,7 +12,7 @@ jobs: name: Preflight steps: - name: Checkout code - uses: actions/checkout@v2 + uses: actions/checkout@v3 with: fetch-depth: 0 @@ -45,7 +45,7 @@ jobs: name: Release ${{ matrix.component }} Docker image steps: - name: Checkout code - uses: actions/checkout@v2 + uses: actions/checkout@v3 with: fetch-depth: 0 @@ -61,14 +61,18 @@ jobs: set -e docker pull opencuebuild/${{ matrix.component }}:${BUILD_ID} - - name: Rebuild and push Docker image - uses: docker/build-push-action@v1 + - name: Login to Docker Hub + uses: docker/login-action@v2 with: username: ${{ secrets.DOCKER_USER }} password: ${{ secrets.DOCKER_PASS }} - dockerfile: ${{ matrix.component }}/Dockerfile - repository: opencue/${{ matrix.component }} - tags: ${{ env.BUILD_ID }}, latest + + - name: Rebuild and push Docker image + uses: docker/build-push-action@v3 + with: + file: ${{ matrix.component }}/Dockerfile + tags: opencue/${{ matrix.component }}:${{ env.BUILD_ID }},opencue/${{ matrix.component }}:latest + push: true create_release: needs: preflight @@ -76,7 +80,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout code - uses: actions/checkout@v2 + uses: actions/checkout@v3 with: fetch-depth: 0 @@ -103,7 +107,7 @@ jobs: run: | mkdir -p "${GITHUB_WORKSPACE}/artifacts/" aws s3 sync "s3://${S3_BUCKET}/opencue/${BUILD_ID}/" "${GITHUB_WORKSPACE}/artifacts/" - echo "::set-output name=filenames::$(ls "${GITHUB_WORKSPACE}/artifacts/" | xargs)" + echo "filenames=$(ls "${GITHUB_WORKSPACE}/artifacts/" | xargs)" >> ${GITHUB_OUTPUT} - name: List artifacts run: | @@ -114,9 +118,12 @@ jobs: run: | last_tagged_version=$(git describe --tags --abbrev=0 $(git rev-list --tags --skip=1 --max-count=1)) commits_since_last_release=$(git log --reverse --pretty="* %H %s" ${last_tagged_version}..HEAD) + # Use a delimiter to preserve the multiline string. # See https://github.community/t/set-output-truncates-multiline-strings/16852 - commits_since_last_release="${commits_since_last_release//$'\n'/'%0A'}" - echo "::set-output name=commits::${commits_since_last_release}" + delimiter="$(openssl rand -hex 8)" + echo "commits<<${delimiter}" >> ${GITHUB_OUTPUT} + echo "${commits_since_last_release}" >> ${GITHUB_OUTPUT} + echo "${delimiter}" >> ${GITHUB_OUTPUT} - name: Create release id: create_release diff --git a/.github/workflows/sonar-cloud-pipeline.yml b/.github/workflows/sonar-cloud-pipeline.yml index ec330da8c..1319f828b 100644 --- a/.github/workflows/sonar-cloud-pipeline.yml +++ b/.github/workflows/sonar-cloud-pipeline.yml @@ -12,7 +12,7 @@ jobs: name: Analyze Python Components steps: - name: Checkout - uses: actions/checkout@v2 + uses: actions/checkout@v3 with: # Fetch all Git history, otherwise the current version number will # not be correctly calculated. @@ -33,7 +33,7 @@ jobs: name: Analyze Cuebot steps: - name: Checkout - uses: actions/checkout@v2 + uses: actions/checkout@v3 with: # Fetch all Git history, otherwise the current version number will # not be correctly calculated. diff --git a/.github/workflows/testing-pipeline.yml b/.github/workflows/testing-pipeline.yml index d3583bd0e..98e5ca166 100644 --- a/.github/workflows/testing-pipeline.yml +++ b/.github/workflows/testing-pipeline.yml @@ -12,7 +12,7 @@ jobs: runs-on: ubuntu-latest container: aswf/ci-opencue:2019 steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: Run Python Tests run: ci/run_python_tests.sh @@ -22,7 +22,7 @@ jobs: container: image: aswf/ci-opencue:2019 steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: Build with Gradle run: | chown -R aswfuser:aswfgroup . @@ -33,7 +33,7 @@ jobs: runs-on: ubuntu-latest container: aswf/ci-opencue:2020 steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: Run Python Tests run: ci/run_python_tests.sh @@ -43,7 +43,7 @@ jobs: container: image: aswf/ci-opencue:2020 steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: Build with Gradle run: | chown -R aswfuser:aswfgroup . @@ -54,7 +54,7 @@ jobs: runs-on: ubuntu-latest container: aswf/ci-opencue:2021 steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: Run Python Tests run: ci/run_python_tests.sh @@ -64,7 +64,7 @@ jobs: container: image: aswf/ci-opencue:2021 steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: Build with Gradle run: | chown -R aswfuser:aswfgroup . @@ -75,7 +75,7 @@ jobs: runs-on: ubuntu-latest container: aswf/ci-opencue:2022 steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: Run Python Tests run: ci/run_python_tests.sh @@ -85,7 +85,7 @@ jobs: container: image: aswf/ci-opencue:2022 steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: Build with Gradle run: | chown -R aswfuser:aswfgroup . @@ -96,7 +96,7 @@ jobs: runs-on: ubuntu-latest container: aswf/ci-opencue:2022 steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: Lint Python Code run: ci/run_python_lint.sh @@ -106,7 +106,7 @@ jobs: container: image: aswf/ci-opencue:2020 steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: Run Sphinx build run: ci/build_sphinx_docs.sh @@ -114,18 +114,18 @@ jobs: name: Check Changed Files runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: Get Changed Files id: get_changed_files - uses: jitterbit/get-changed-files@v1 + uses: tj-actions/changed-files@v35 - name: Check for Version Change - run: ci/check_changed_files.py ${{ steps.get_changed_files.outputs.modified }} ${{ steps.get_changed_files.outputs.removed }} + run: ci/check_changed_files.py ${{ steps.get_changed_files.outputs.modified_files }} ${{ steps.get_changed_files.outputs.deleted_files }} check_migration_files: name: Check Database Migration Files runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: Check Migration Files run: ci/check_database_migrations.py @@ -133,9 +133,9 @@ jobs: name: Check for Version Bump runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: Get Changed Files id: get_changed_files - uses: jitterbit/get-changed-files@v1 + uses: tj-actions/changed-files@v35 - name: Check for Version Change - run: ci/check_version_bump.py ${{ steps.get_changed_files.outputs.all }} + run: ci/check_version_bump.py ${{ steps.get_changed_files.outputs.all_changed_and_modified_files }} From df96a2f962ec14ad1e004c241891037e734c01c1 Mon Sep 17 00:00:00 2001 From: Brian Cipriano Date: Fri, 20 Jan 2023 15:06:19 -0500 Subject: [PATCH 12/56] Fix Docker build in the packaging pipeline. (#1250) --- .github/workflows/packaging-pipeline.yml | 1 + .github/workflows/release-pipeline.yml | 1 + 2 files changed, 2 insertions(+) diff --git a/.github/workflows/packaging-pipeline.yml b/.github/workflows/packaging-pipeline.yml index 6fcfa5676..6e21eab65 100644 --- a/.github/workflows/packaging-pipeline.yml +++ b/.github/workflows/packaging-pipeline.yml @@ -76,6 +76,7 @@ jobs: with: file: ${{ matrix.component }}/Dockerfile tags: opencuebuild/${{ matrix.component }}:${{ env.BUILD_ID }} + context: . push: true - name: Extract Artifacts diff --git a/.github/workflows/release-pipeline.yml b/.github/workflows/release-pipeline.yml index 1111bec54..48aeb9f5a 100644 --- a/.github/workflows/release-pipeline.yml +++ b/.github/workflows/release-pipeline.yml @@ -72,6 +72,7 @@ jobs: with: file: ${{ matrix.component }}/Dockerfile tags: opencue/${{ matrix.component }}:${{ env.BUILD_ID }},opencue/${{ matrix.component }}:latest + context: . push: true create_release: From 04660512566565814a1b36a975ac77e7a1cdfea5 Mon Sep 17 00:00:00 2001 From: Brian Cipriano Date: Fri, 20 Jan 2023 23:46:42 -0500 Subject: [PATCH 13/56] Lock evdev dependency for Python 2. (#1248) --- ci/run_python_tests.sh | 2 +- requirements.txt | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/ci/run_python_tests.sh b/ci/run_python_tests.sh index c37b539ad..c6a51a015 100755 --- a/ci/run_python_tests.sh +++ b/ci/run_python_tests.sh @@ -2,7 +2,7 @@ set -e -python_version=$(python -V) +python_version=$(python -V 2>&1) echo "Will run tests using ${python_version}" pip install --user -r requirements.txt -r requirements_gui.txt diff --git a/requirements.txt b/requirements.txt index 92b375d60..44adbe5b8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,6 @@ 2to3==1.0 enum34==1.1.6 +evdev==1.4.0;python_version<"3.0" and "linux" in sys_platform future==0.17.1 futures==3.2.0;python_version<"3.0" grpcio==1.26.0;python_version<"3.0" From 633908735f81aa1e503c1618ddb1b8eb7d058f4d Mon Sep 17 00:00:00 2001 From: Brian Cipriano Date: Fri, 20 Jan 2023 23:47:51 -0500 Subject: [PATCH 14/56] [rqd] Raise exception during startup if CUEBOT_HOSTNAME is empty. (#1237) --- rqd/rqd/rqnetwork.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/rqd/rqd/rqnetwork.py b/rqd/rqd/rqnetwork.py index 9de2cf452..9f33e64a2 100644 --- a/rqd/rqd/rqnetwork.py +++ b/rqd/rqd/rqnetwork.py @@ -38,6 +38,7 @@ import rqd.compiled_proto.report_pb2_grpc import rqd.compiled_proto.rqd_pb2_grpc import rqd.rqconstants +import rqd.rqexceptions import rqd.rqdservicers import rqd.rqutil @@ -278,12 +279,13 @@ def __getChannel(self): ), ) - cuebots = rqd.rqconstants.CUEBOT_HOSTNAME.split() + cuebots = rqd.rqconstants.CUEBOT_HOSTNAME.strip().split() + if len(cuebots) == 0: + raise rqd.rqexceptions.RqdException("CUEBOT_HOSTNAME is empty") shuffle(cuebots) - if len(cuebots) > 0: - self.channel = grpc.insecure_channel('%s:%s' % (cuebots[0], - rqd.rqconstants.CUEBOT_GRPC_PORT)) - self.channel = grpc.intercept_channel(self.channel, *interceptors) + self.channel = grpc.insecure_channel('%s:%s' % (cuebots[0], + rqd.rqconstants.CUEBOT_GRPC_PORT)) + self.channel = grpc.intercept_channel(self.channel, *interceptors) atexit.register(self.closeChannel) def __getReportStub(self): From 5d5ef8f73d03f43b8a9c955f6cd0a9173f4b6f82 Mon Sep 17 00:00:00 2001 From: Brian Cipriano Date: Fri, 20 Jan 2023 23:59:50 -0500 Subject: [PATCH 15/56] [cuebot] Fix a few database query and test issues. (#1232) --- .../spcue/dao/postgres/ProcDaoJdbc.java | 39 ++++++++----------- .../dao/postgres/DispatcherDaoFifoTests.java | 1 - .../test/dao/postgres/LayerDaoTests.java | 24 +++++++----- 3 files changed, 32 insertions(+), 32 deletions(-) diff --git a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/ProcDaoJdbc.java b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/ProcDaoJdbc.java index b68f8a74c..5af292fb3 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/ProcDaoJdbc.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/ProcDaoJdbc.java @@ -262,30 +262,25 @@ public void updateProcMemoryUsage(FrameInterface f, long rss, long maxRss, "SELECT pk_frame FROM proc WHERE pk_frame=? FOR UPDATE", String.class, f.getFrameId()).equals(f.getFrameId())) { - getJdbcTemplate().update(UPDATE_PROC_MEMORY_USAGE, - rss, maxRss, vss, maxVss, - usedGpuMemory, maxUsedGpuMemory, f.getFrameId()); - } getJdbcTemplate().update(new PreparedStatementCreator() { - @Override - public PreparedStatement createPreparedStatement(Connection conn) - throws SQLException { - PreparedStatement updateProc = conn.prepareStatement( - UPDATE_PROC_MEMORY_USAGE); - updateProc.setLong(1, rss); - updateProc.setLong(2, maxRss); - updateProc.setLong(3, vss); - updateProc.setLong(4, maxVss); - updateProc.setLong(5, usedGpuMemory); - updateProc.setLong(6, maxUsedGpuMemory); - updateProc.setBytes(7, children); - updateProc.setString(8, f.getFrameId()); - return updateProc; - } - } - ); + @Override + public PreparedStatement createPreparedStatement(Connection conn) + throws SQLException { + PreparedStatement updateProc = conn.prepareStatement( + UPDATE_PROC_MEMORY_USAGE); + updateProc.setLong(1, rss); + updateProc.setLong(2, maxRss); + updateProc.setLong(3, vss); + updateProc.setLong(4, maxVss); + updateProc.setLong(5, usedGpuMemory); + updateProc.setLong(6, maxUsedGpuMemory); + updateProc.setBytes(7, children); + updateProc.setString(8, f.getFrameId()); + return updateProc; + } + }); } - catch (DataAccessException dae) { + } catch (DataAccessException dae) { logger.info("The proc for frame " + f + " could not be updated with new memory stats: " + dae); } diff --git a/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/DispatcherDaoFifoTests.java b/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/DispatcherDaoFifoTests.java index 5de19273e..c34396709 100644 --- a/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/DispatcherDaoFifoTests.java +++ b/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/DispatcherDaoFifoTests.java @@ -224,7 +224,6 @@ public void testFifoSchedulingDisabled() throws Exception { List sortedJobs = new ArrayList(jobs); Collections.sort(sortedJobs, Comparator.comparing(jobId -> jobManager.getJob(jobId).getName())); - assertNotEquals(jobs, sortedJobs); for (int i = 0; i < count; i++) { assertEquals("pipe-default-testuser_job" + i, diff --git a/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/LayerDaoTests.java b/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/LayerDaoTests.java index 83c8a466b..189178689 100644 --- a/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/LayerDaoTests.java +++ b/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/LayerDaoTests.java @@ -20,6 +20,7 @@ package com.imageworks.spcue.test.dao.postgres; import java.io.File; +import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.HashMap; @@ -44,7 +45,6 @@ import com.imageworks.spcue.LayerDetail; import com.imageworks.spcue.LayerInterface; import com.imageworks.spcue.LimitEntity; -import com.imageworks.spcue.LimitInterface; import com.imageworks.spcue.ResourceUsage; import com.imageworks.spcue.config.TestAppConfig; import com.imageworks.spcue.dao.DepartmentDao; @@ -63,9 +63,11 @@ import com.imageworks.spcue.util.FrameSet; import com.imageworks.spcue.util.JobLogUtil; +import static org.hamcrest.Matchers.containsInAnyOrder; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertThat; import static org.junit.Assert.assertTrue; @Transactional @@ -116,7 +118,11 @@ public void testMode() { } public LayerDetail getLayer() { + List layers = getLayers(); + return layers.get(layers.size()-1); + } + public List getLayers() { JobSpec spec = jobLauncher.parse(new File("src/test/resources/conf/jobspec/jobspec.xml")); JobDetail job = spec.getJobs().get(0).detail; job.groupId = ROOT_FOLDER; @@ -126,14 +132,13 @@ public LayerDetail getLayer() { job.facilityId = facilityDao.getDefaultFacility().getId(); jobDao.insertJob(job, jobLogUtil); - LayerDetail lastLayer= null; + List result = new ArrayList<>(); String limitId = limitDao.createLimit(LIMIT_NAME, LIMIT_MAX_VALUE); limitDao.createLimit(LIMIT_TEST_A, 1); limitDao.createLimit(LIMIT_TEST_B, 2); limitDao.createLimit(LIMIT_TEST_C, 3); for (BuildableLayer buildableLayer: spec.getJobs().get(0).getBuildableLayers()) { - LayerDetail layer = buildableLayer.layerDetail; FrameSet frameSet = new FrameSet(layer.range); int num_frames = frameSet.size(); @@ -147,10 +152,10 @@ public LayerDetail getLayer() { layerDao.insertLayerDetail(layer); layerDao.insertLayerEnvironment(layer, buildableLayer.env); layerDao.addLimit(layer, limitId); - lastLayer = layer; + result.add(layer); } - return lastLayer; + return result; } public JobDetail getJob() { @@ -202,16 +207,17 @@ public void testGetLayerDetail() { LayerDetail l2 = layerDao.getLayerDetail(layer); LayerDetail l3 = layerDao.getLayerDetail(layer.id); - assertEquals(l2, l3); + assertEquals(layer, l2); + assertEquals(layer, l3); } @Test @Transactional @Rollback(true) public void testGetLayerDetails() { - LayerDetail layer = getLayer(); - List ld = layerDao.getLayerDetails(getJob()); - assertEquals(ld.get(0).name, LAYER_NAME); + List wantLayers = getLayers(); + List gotLayers = layerDao.getLayerDetails(getJob()); + assertThat(gotLayers, containsInAnyOrder(wantLayers.toArray())); } @Test From f7cf0bdea1e75c9c56da7ae5c7ccdf7a5aab5677 Mon Sep 17 00:00:00 2001 From: Brian Cipriano Date: Sat, 21 Jan 2023 00:06:19 -0500 Subject: [PATCH 16/56] Create initial integration test script. (#1245) --- .github/workflows/packaging-pipeline.yml | 22 ++- ci/run_integration_test.sh | 234 +++++++++++++++++++++++ sandbox/flyway.Dockerfile | 6 +- sandbox/install-client-sources.sh | 9 +- 4 files changed, 261 insertions(+), 10 deletions(-) create mode 100755 ci/run_integration_test.sh diff --git a/.github/workflows/packaging-pipeline.yml b/.github/workflows/packaging-pipeline.yml index 6e21eab65..c7ab1f85c 100644 --- a/.github/workflows/packaging-pipeline.yml +++ b/.github/workflows/packaging-pipeline.yml @@ -6,8 +6,25 @@ on: branches: [ master ] jobs: - build_components: + integration_test: + name: Run Integration Test runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v3 + + - name: Run test + run: ci/run_integration_test.sh + + - name: Archive log files + uses: actions/upload-artifact@v3 + if: ${{ always() }} + with: + name: test-logs + path: /tmp/opencue-test/*.log + + build_components: + needs: integration_test strategy: matrix: component: [cuebot, rqd] @@ -41,6 +58,7 @@ jobs: ARTIFACTS: cueadmin-${BUILD_ID}-all.tar.gz name: Build ${{ matrix.NAME }} + runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v3 @@ -101,9 +119,9 @@ jobs: done create_other_artifacts: + name: Create Other Build Artifacts needs: build_components runs-on: ubuntu-latest - name: Create Other Build Artifacts steps: - name: Checkout uses: actions/checkout@v3 diff --git a/ci/run_integration_test.sh b/ci/run_integration_test.sh new file mode 100755 index 000000000..ee0a37fe6 --- /dev/null +++ b/ci/run_integration_test.sh @@ -0,0 +1,234 @@ +#!/bin/bash + +set -e + +RQD_ROOT="/tmp/rqd" +TEST_LOGS="/tmp/opencue-test" +DOCKER_COMPOSE_LOG="${TEST_LOGS}/docker-compose.log" +DB_DATA_DIR="sandbox/db-data" +VENV="/tmp/opencue-integration-venv" + +log() { + echo "$(date "+%Y-%m-%d %H:%M:%S") $1 $2" +} + +kill_descendant_processes() { + local pid="$1" + local and_self="${2:-false}" + if children="$(pgrep -P "$pid")"; then + for child in $children; do + kill_descendant_processes "$child" true + done + fi + if [[ "$and_self" == true ]]; then + kill "$pid" 2>/dev/null || true + fi +} + +verify_command_exists() { + if ! command -v $1 &> /dev/null; then + log ERROR "command \"$1\" was not found" + exit 1 + fi +} + +verify_no_database() { + if [ -e "${DB_DATA_DIR}" ]; then + log ERROR "Postgres data directory ${DB_DATA_DIR} already exists" + exit 1 + fi +} + +verify_no_containers() { + num_containers=$(docker compose ps --format json | jq length) + if [[ $num_containers -gt 0 ]]; then + log ERROR "Found ${num_containers} Docker compose containers, clean these up with \`docker compose rm\` before continuing" + exit 1 + fi +} + +create_rqd_root() { + if [ -e "$RQD_ROOT" ]; then + log ERROR "log root ${RQD_ROOT} already exists" + exit 1 + fi + + mkdir -p "${RQD_ROOT}/logs" + mkdir "${RQD_ROOT}/shots" +} + +wait_for_service_state() { + log INFO "Waiting for service \"$1\" to have state \"$2\"..." + while true; do + current_time=$(date +%s) + if [[ $current_time -gt $3 ]]; then + log ERROR "Timed out waiting for Docker compose to come up" + exit 1 + fi + container=$(docker compose ps --all --format json | jq ".[] | select(.Service==\"$1\")") + if [[ ${container} = "" ]]; then + log INFO "Service \"$1\": no container yet" + else + container_name=$(echo "$container" | jq -r '.Name') + current_state=$(echo "$container" | jq -r '.State') + log INFO "Service \"$1\": container \"${container_name}\" state = ${current_state}" + if [[ ${current_state} = $2 ]]; then + break + fi + fi + sleep 5 + done +} + +verify_flyway_success() { + container=$(docker compose ps --all --format json | jq '.[] | select(.Service=="flyway")') + container_name=$(echo "$container" | jq -r '.Name') + exit_code=$(echo "$container" | jq -r '.ExitCode') + if [[ ${exit_code} = 0 ]]; then + log INFO "Service \"flyway\": container \"${container_name}\" exit code = 0 (PASS)" + else + log ERROR "Service \"flyway\": container \"${container_name}\" exit code = ${exit_code} (FAIL)" + exit 1 + fi +} + +verify_migration_versions() { + migrations_in_db=$(docker compose exec -e PGUSER=cuebot db psql -Aqtc "SELECT COUNT(*) FROM flyway_schema_history") + migrations_in_code=$(ls cuebot/src/main/resources/conf/ddl/postgres/migrations/ | wc -l | tr -d ' ') + if [[ ${migrations_in_db} = ${migrations_in_code} ]]; then + log INFO "Database and code both contain ${migrations_in_db} migrations (PASS)" + else + log ERROR "Database contains ${migrations_in_db} migrations, code contains ${migrations_in_code} (FAIL)" + exit 1 + fi +} + +create_and_activate_venv() { + if [[ -d "${VENV}" ]]; then + rm -rf "${VENV}" + fi + python3 -m venv "${VENV}" + source "${VENV}/bin/activate" +} + +test_pycue() { + want_shows="['testing']" + got_shows=$(python -c 'import opencue; print([show.name() for show in opencue.api.getShows()])') + if [[ "${got_shows}" = "${want_shows}" ]]; then + log INFO "(pycue) Got expected show list (PASS)" + else + log ERROR "(pycue) Got unexpected show list (FAIL)" + log ERROR "got: ${got_shows}, want: ${want_shows}" + exit 1 + fi + + rqd_name=$(docker compose ps --format json | jq -r '.[] | select(.Service=="rqd") | .Name') + rqd_ip=$(docker inspect -f '{{range.NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "${rqd_name}") + want_hosts="['${rqd_ip}']" + got_hosts=$(python -c 'import opencue; print([host.name() for host in opencue.api.getHosts()])') + if [[ "${got_hosts}" = "${want_hosts}" ]]; then + log INFO "(pycue) Got expected host list (PASS)" + else + log ERROR "(pycue) Got unexpected host list (FAIL)" + log ERROR "got: ${got_hosts}, want: ${want_hosts}" + exit 1 + fi +} + +test_cueadmin() { + want_show="testing" + ls_response=$(cueadmin -ls) + got_show=$(echo "${ls_response}" | tail -n 1 | cut -d ' ' -f 1) + if [[ "${got_show}" = "${want_show}" ]]; then + log INFO "(cueadmin) Got expected -ls response (PASS)" + else + log ERROR "(cueadmin) Got unexpected -ls response (FAIL)" + log ERROR "got show: ${got_show}, want show: ${want_show}" + log ERROR "full response: ${ls_response}" + exit 1 + fi + + rqd_name=$(docker compose ps --format json | jq -r '.[] | select(.Service=="rqd") | .Name') + want_host=$(docker inspect -f '{{range.NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "${rqd_name}") + lh_response=$(cueadmin -lh) + got_host=$(echo "${lh_response}" | tail -n 1 | cut -d ' ' -f 1) + if [[ "${got_host}" = "${want_host}" ]]; then + log INFO "(cueadmin) Got expected -lh response (PASS)" + else + log ERROR "(cueadmin) Got unexpected -lh response (FAIL)" + log ERROR "got host: ${got_host}, want host: ${want_host}" + log ERROR "full response: ${lh_response}" + exit 1 + fi +} + +cleanup() { + docker compose rm --stop --force >>"${DOCKER_COMPOSE_LOG}" 2>&1 + rm -rf "${RQD_ROOT}" || true + rm -rf "${DB_DATA_DIR}" || true + rm -rf "${VENV}" || true +} + +main() { + # Ensure all subshells in the background are terminated when the main script exits. + trap "{ kill_descendant_processes $$; exit; }" SIGINT SIGTERM EXIT + + mkdir -p "${TEST_LOGS}" + if [[ "${CI:-false}" == true ]]; then + log INFO "More logs can be found under the test-logs artifact attached to this workflow execution" + else + log INFO "More logs can be found at ${TEST_LOGS}" + fi + + CI_DIRECTORY=$(cd "$(dirname "${BASH_SOURCE[0]}")" &> /dev/null && pwd) + OPENCUE_ROOT=$(dirname "${CI_DIRECTORY}") + log INFO "OpenCue project is located at ${OPENCUE_ROOT}" + cd "${OPENCUE_ROOT}" + + verify_command_exists docker + verify_command_exists "docker compose" + verify_command_exists jq + verify_no_database + verify_no_containers + create_rqd_root + + log INFO "$(docker --version)" + log INFO "$(docker compose version)" + + log INFO "Building Cuebot image..." + docker build -t opencue/cuebot -f cuebot/Dockerfile . &>"${TEST_LOGS}/docker-build-cuebot.log" + log INFO "Building RQD image..." + docker build -t opencue/rqd -f rqd/Dockerfile . &>"${TEST_LOGS}/docker-build-rqd.log" + + log INFO "Starting Docker compose..." + docker compose up &>"${DOCKER_COMPOSE_LOG}" & + if [[ "$(uname -s)" == "Darwin" ]]; then + docker_timeout=$(date -v +5M +%s) + else + docker_timeout=$(date -d '5 min' +%s) + fi + wait_for_service_state "db" "running" $docker_timeout + wait_for_service_state "flyway" "exited" $docker_timeout + wait_for_service_state "cuebot" "running" $docker_timeout + wait_for_service_state "rqd" "running" $docker_timeout + + verify_flyway_success + verify_migration_versions + log INFO "Creating Python virtual environment..." + create_and_activate_venv + log INFO "Installing OpenCue Python libraries..." + install_log="${TEST_LOGS}/install-client-sources.log" + sandbox/install-client-sources.sh &>"${install_log}" + log INFO "Testing pycue library..." + test_pycue + log INFO "Testing cueadmin..." + test_cueadmin + + # TODO Launch a job and verify it finishes. + + cleanup + + log INFO "Success" +} + +main diff --git a/sandbox/flyway.Dockerfile b/sandbox/flyway.Dockerfile index 086d47bbd..fffcbd3c2 100644 --- a/sandbox/flyway.Dockerfile +++ b/sandbox/flyway.Dockerfile @@ -1,11 +1,11 @@ FROM almalinux:8.7 -ARG FLYWAY_VERSION=9.9.0 +ARG FLYWAY_VERSION=9.11.0 # Get flyway RUN yum install -y tar java-1.8.0-openjdk postgresql-jdbc nc postgresql -RUN curl -O https://repo1.maven.org/maven2/org/flywaydb/flyway-commandline/${FLYWAY_VERSION}/flyway-commandline-${FLYWAY_VERSION}-linux-x64.tar.gz -RUN tar -xzf flyway-commandline-${FLYWAY_VERSION}-linux-x64.tar.gz +RUN curl -O https://repo1.maven.org/maven2/org/flywaydb/flyway-commandline/${FLYWAY_VERSION}/flyway-commandline-${FLYWAY_VERSION}.tar.gz +RUN tar -xzf flyway-commandline-${FLYWAY_VERSION}.tar.gz WORKDIR flyway-${FLYWAY_VERSION} diff --git a/sandbox/install-client-sources.sh b/sandbox/install-client-sources.sh index 1030f492b..7e15ed018 100755 --- a/sandbox/install-client-sources.sh +++ b/sandbox/install-client-sources.sh @@ -4,14 +4,13 @@ set -e pip install -r requirements.txt -r requirements_gui.txt -# Compile the proto used to communicate with the Cuebot server +# Compile the proto used to communicate with the Cuebot server. cd proto python -m grpc_tools.protoc -I=. \ --python_out=../pycue/opencue/compiled_proto \ --grpc_python_out=../pycue/opencue/compiled_proto ./*.proto cd .. +2to3 -wn -f import pycue/opencue/compiled_proto/*_pb2*.py -# Install the OpenCue client packages -# You also need to set the OL_CONFIG environment variable -# to pyoutline/etc/outline.cfg to run Cuesubmit -pip install pycue/ pyoutline/ cuesubmit/ cuegui/ cueadmin/ +# Install all client packages. +pip install pycue/ pyoutline/ cueadmin/ cuesubmit/ cuegui/ From 8a33d8394ec390e75761f0798d3b403a3b7e7936 Mon Sep 17 00:00:00 2001 From: Brian Cipriano Date: Thu, 26 Jan 2023 18:23:09 -0500 Subject: [PATCH 17/56] [cuegui] Move constants to a YAML config file. (#1242) --- cuebot/src/main/resources/opencue.properties | 3 +- cuegui/cuegui/Constants.py | 214 +++++++++++------- cuegui/cuegui/{Config.py => Layout.py} | 5 +- cuegui/cuegui/Main.py | 4 +- cuegui/cuegui/Redirect.py | 2 +- cuegui/cuegui/Utils.py | 18 +- cuegui/cuegui/config/cue_resources.yaml | 28 --- cuegui/cuegui/config/cuegui.yaml | 112 +++++++++ cuegui/tests/Constants_tests.py | 166 ++++++++++++++ .../{Config_tests.py => Layout_tests.py} | 18 +- cuegui/tests/Utils_tests.py | 12 + 11 files changed, 442 insertions(+), 140 deletions(-) rename cuegui/cuegui/{Config.py => Layout.py} (94%) delete mode 100644 cuegui/cuegui/config/cue_resources.yaml create mode 100644 cuegui/cuegui/config/cuegui.yaml create mode 100644 cuegui/tests/Constants_tests.py rename cuegui/tests/{Config_tests.py => Layout_tests.py} (86%) diff --git a/cuebot/src/main/resources/opencue.properties b/cuebot/src/main/resources/opencue.properties index 145bbb352..a08522eb1 100644 --- a/cuebot/src/main/resources/opencue.properties +++ b/cuebot/src/main/resources/opencue.properties @@ -144,8 +144,7 @@ protected_shows=testing # -1 means shows should not get deactivated at all. max_show_stale_days=-1 -# These flags determine whether or not layers/frames will be readonly when job is finished. +# These flags determine whether layers/frames will be readonly when job is finished. # If flags are set as true, layers/frames cannot be retried, eaten, edited dependency on, etc. -# In order to toggle the same functionility on cuegui side, set flags in cue_resources.yaml layer.finished_jobs_readonly=false frame.finished_jobs_readonly=false \ No newline at end of file diff --git a/cuegui/cuegui/Constants.py b/cuegui/cuegui/Constants.py index a988538c3..4dcef529d 100644 --- a/cuegui/cuegui/Constants.py +++ b/cuegui/cuegui/Constants.py @@ -17,91 +17,165 @@ Application constants. """ - from __future__ import print_function from __future__ import division from __future__ import absolute_import +import logging import os import platform -from PySide2 import QtCore from PySide2 import QtGui from PySide2 import QtWidgets +import yaml import opencue +import opencue.config -possible_version_path = os.path.join( - os.path.abspath(os.path.join(__file__ , "../../..")), 'VERSION.in') -if os.path.exists(possible_version_path): - with open(possible_version_path) as fp: - VERSION = fp.read().strip() -else: - VERSION = "1.3.0" +__CONFIG_FILE_ENV_VAR = 'CUEGUI_CONFIG_FILE' +__DEFAULT_INI_PATH_ENV_VAR = 'CUEGUI_DEFAULT_INI_PATH' +__DEFAULT_CONFIG_FILE_NAME = 'cuegui.yaml' +__DEFAULT_CONFIG_FILE = os.path.join( + os.path.dirname(__file__), 'config', __DEFAULT_CONFIG_FILE_NAME) + + +def __getLogger(): + """Other code should use cuegui.Logger to get a logger; we avoid using that module here + to avoid creating a circular dependency.""" + logger_format = logging.Formatter("%(levelname)-9s %(module)-10s %(message)s") + logger_stream = logging.StreamHandler() + logger_stream.setLevel(logging.INFO) + logger_stream.setFormatter(logger_format) + logger = logging.getLogger(__file__) + logger.addHandler(logger_stream) + return logger + + +def __loadConfigFromFile(): + logger = __getLogger() + with open(__DEFAULT_CONFIG_FILE) as fp: + config = yaml.load(fp, Loader=yaml.SafeLoader) + + user_config_file = None + + logger.debug('Checking for cuegui config file path in %s', __CONFIG_FILE_ENV_VAR) + config_file_from_env = os.environ.get(__CONFIG_FILE_ENV_VAR) + if config_file_from_env and os.path.exists(config_file_from_env): + user_config_file = config_file_from_env + + if not user_config_file: + config_file_from_user_profile = os.path.join( + opencue.config.config_base_directory(), __DEFAULT_CONFIG_FILE_NAME) + logger.debug('Checking for cuegui config at %s', config_file_from_user_profile) + if os.path.exists(config_file_from_user_profile): + user_config_file = config_file_from_user_profile + + if user_config_file: + logger.info('Loading cuegui config from %s', user_config_file) + with open(user_config_file, 'r') as fp: + config.update(yaml.load(fp, Loader=yaml.SafeLoader)) + + return config + + +def __packaged_version(): + possible_version_path = os.path.join( + os.path.abspath(os.path.join(__file__, "../../..")), 'VERSION.in') + if os.path.exists(possible_version_path): + with open(possible_version_path) as fp: + default_version = fp.read().strip() + return default_version + return "1.3.0" -STARTUP_NOTICE_DATE = 0 -STARTUP_NOTICE_MSG = "" -JOB_UPDATE_DELAY = 10000 # msec -LAYER_UPDATE_DELAY = 10000 # msec -FRAME_UPDATE_DELAY = 10000 # msec -HOST_UPDATE_DELAY = 20000 # msec -AFTER_ACTION_UPDATE_DELAY = 1000 # msec +__config = __loadConfigFromFile() -MAX_LOG_POPUPS = 5 -MINIMUM_UPDATE_INTERVAL = 5 # sec +VERSION = __config.get('version', __packaged_version()) -FONT_SIZE = 10 # 8 -STANDARD_FONT = QtGui.QFont("Luxi Sans", FONT_SIZE) -STANDARD_ROW_HEIGHT = 16 # 14 +STARTUP_NOTICE_DATE = __config.get('startup_notice.date') +STARTUP_NOTICE_MSG = __config.get('startup_notice.msg') -MEMORY_WARNING_LEVEL = 5242880 +JOB_UPDATE_DELAY = __config.get('refresh.job_update_delay') +LAYER_UPDATE_DELAY = __config.get('refresh.layer_update_delay') +FRAME_UPDATE_DELAY = __config.get('refresh.frame_update_delay') +HOST_UPDATE_DELAY = __config.get('refresh.host_update_delay') +AFTER_ACTION_UPDATE_DELAY = __config.get('refresh.after_action_update_delay') +MINIMUM_UPDATE_INTERVAL = __config.get('refresh.min_update_interval') // 1000 -RESOURCE_PATH = os.path.dirname(__file__) + "/images" -DEFAULT_INI_PATH = os.getenv('CUEGUI_DEFAULT_INI_PATH', os.path.dirname(__file__) + '/config') +FONT_FAMILY = __config.get('style.font.family') +FONT_SIZE = __config.get('style.font.size') +STANDARD_FONT = QtGui.QFont(FONT_FAMILY, FONT_SIZE) -DEFAULT_PLUGIN_PATHS = [os.path.dirname(__file__) + "/plugins"] +RESOURCE_PATH = __config.get('paths.resources') +if not os.path.isabs(RESOURCE_PATH): + RESOURCE_PATH = os.path.abspath(os.path.join(os.path.dirname(__file__), RESOURCE_PATH)) -LOGGER_FORMAT = "%(levelname)-9s %(module)-10s %(message)s" -LOGGER_LEVEL = "WARNING" +CONFIG_PATH = __config.get('paths.config') +if not os.path.isabs(CONFIG_PATH): + CONFIG_PATH = os.path.abspath(os.path.join(os.path.dirname(__file__), CONFIG_PATH)) -EMAIL_SUBJECT_PREFIX = "cuemail: please check " -EMAIL_BODY_PREFIX = "Your PSTs request that you check " -EMAIL_BODY_SUFFIX = "\n\n" -EMAIL_DOMAIN = "" +DEFAULT_INI_PATH = os.getenv('CUEGUI_DEFAULT_INI_PATH', __config.get('paths.default_ini_path')) +if not os.path.isabs(DEFAULT_INI_PATH): + DEFAULT_INI_PATH = os.path.abspath(os.path.join(os.path.dirname(__file__), DEFAULT_INI_PATH)) -GITHUB_CREATE_ISSUE_URL = 'https://github.com/AcademySoftwareFoundation/OpenCue/issues/new' -URL_USERGUIDE = "https://www.opencue.io/docs/" -URL_SUGGESTION = "%s?labels=enhancement&template=enhancement.md" % GITHUB_CREATE_ISSUE_URL -URL_BUG = "%s?labels=bug&template=bug_report.md" % GITHUB_CREATE_ISSUE_URL +DEFAULT_PLUGIN_PATHS = __config.get('paths.plugins') +for i, path in enumerate(DEFAULT_PLUGIN_PATHS): + if not os.path.isabs(path): + DEFAULT_PLUGIN_PATHS[i] = os.path.abspath(os.path.join(os.path.dirname(__file__), path)) -if platform.system() == "Windows": - DEFAULT_EDITOR = "notepad" +LOGGER_FORMAT = __config.get('logger.format') +LOGGER_LEVEL = __config.get('logger.level') + +EMAIL_SUBJECT_PREFIX = __config.get('email.subject_prefix') +EMAIL_BODY_PREFIX = __config.get('email.body_prefix') +EMAIL_BODY_SUFFIX = __config.get('email.body_suffix') +EMAIL_DOMAIN = __config.get('email.domain') + +GITHUB_CREATE_ISSUE_URL = __config.get('links.issue.create') +URL_USERGUIDE = __config.get('links.user_guide') +URL_SUGGESTION = GITHUB_CREATE_ISSUE_URL + __config.get('links.issue.suggestion') +URL_BUG = GITHUB_CREATE_ISSUE_URL + __config.get('links.issue.bug') + +if platform.system() == 'Windows': + DEFAULT_EDITOR = __config.get('editor.windows') +elif platform.system() == 'Darwin': + DEFAULT_EDITOR = __config.get('editor.mac') else: - DEFAULT_EDITOR = "gview -R -m -M -U %s/gvimrc +" % DEFAULT_INI_PATH + DEFAULT_EDITOR = __config.get('editor.linux') +DEFAULT_EDITOR = DEFAULT_EDITOR.format(config_path=CONFIG_PATH) + +LOG_ROOT_OS = __config.get('render_logs.root') + +ALLOWED_TAGS = tuple(__config.get('allowed_tags')) + +DARK_STYLE_SHEET = os.path.join(CONFIG_PATH, __config.get('style.style_sheet')) +COLOR_THEME = __config.get('style.color_theme') +__bg_colors = __config.get('style.colors.background') +COLOR_USER_1 = QtGui.QColor(*__bg_colors[0]) +COLOR_USER_2 = QtGui.QColor(*__bg_colors[1]) +COLOR_USER_3 = QtGui.QColor(*__bg_colors[2]) +COLOR_USER_4 = QtGui.QColor(*__bg_colors[3]) + +__frame_colors = __config.get('style.colors.frame_state') +RGB_FRAME_STATE = { + opencue.api.job_pb2.DEAD: QtGui.QColor(*__frame_colors.get('DEAD')), + opencue.api.job_pb2.DEPEND: QtGui.QColor(*__frame_colors.get('DEPEND')), + opencue.api.job_pb2.EATEN: QtGui.QColor(*__frame_colors.get('EATEN')), + opencue.api.job_pb2.RUNNING: QtGui.QColor(*__frame_colors.get('RUNNING')), + opencue.api.job_pb2.SETUP: QtGui.QColor(*__frame_colors.get('SETUP')), + opencue.api.job_pb2.SUCCEEDED: QtGui.QColor(*__frame_colors.get('SUCCEEDED')), + opencue.api.job_pb2.WAITING: QtGui.QColor(*__frame_colors.get('WAITING')), + opencue.api.job_pb2.CHECKPOINT: QtGui.QColor(*__frame_colors.get('CHECKPOINT')), +} -EMPTY_INDEX = QtCore.QModelIndex() +MEMORY_WARNING_LEVEL = __config.get('memory_warning_level') -QVARIANT_CENTER = QtCore.Qt.AlignCenter -QVARIANT_RIGHT = QtCore.Qt.AlignRight -QVARIANT_NULL = None -QVARIANT_BLACK = QtGui.QColor(QtCore.Qt.black) -QVARIANT_GREY = QtGui.QColor(QtCore.Qt.gray) - -ALLOWED_TAGS = ("general", "desktop", "playblast", "util", "preprocess", "wan", "cuda", "splathw", - 'naiad', 'massive') - -RGB_FRAME_STATE = {opencue.api.job_pb2.DEAD: QtGui.QColor(255, 0, 0), - opencue.api.job_pb2.DEPEND: QtGui.QColor(160, 32, 240), - opencue.api.job_pb2.EATEN: QtGui.QColor(150, 0, 0), - opencue.api.job_pb2.RUNNING: QtGui.QColor(200, 200, 55), - opencue.api.job_pb2.SETUP: QtGui.QColor(160, 32, 240), - opencue.api.job_pb2.SUCCEEDED: QtGui.QColor(55, 200, 55), - opencue.api.job_pb2.WAITING: QtGui.QColor(135, 207, 235), - opencue.api.job_pb2.CHECKPOINT: QtGui.QColor(61, 98, 247)} -QVARIANT_FRAME_STATE = \ - dict((key, RGB_FRAME_STATE[key]) for key in list(RGB_FRAME_STATE.keys())) +LOG_HIGHLIGHT_ERROR = __config.get('render_logs.highlight.error') +LOG_HIGHLIGHT_WARN = __config.get('render_logs.highlight.warning') +LOG_HIGHLIGHT_INFO = __config.get('render_logs.highlight.info') + +RESOURCE_LIMITS = __config.get('resources') TYPE_JOB = QtWidgets.QTreeWidgetItem.UserType + 1 TYPE_LAYER = QtWidgets.QTreeWidgetItem.UserType + 2 @@ -120,27 +194,7 @@ TYPE_TASK = QtWidgets.QTreeWidgetItem.UserType + 15 TYPE_LIMIT = QtWidgets.QTreeWidgetItem.UserType + 16 -COLUMN_INFO_DISPLAY = 2 - -DARK_STYLE_SHEET = os.path.join(DEFAULT_INI_PATH, "darkpalette.qss") -COLOR_THEME = "plastique" -COLOR_USER_1 = QtGui.QColor(50, 50, 100) -COLOR_USER_2 = QtGui.QColor(100, 100, 50) -COLOR_USER_3 = QtGui.QColor(0, 50, 0) -COLOR_USER_4 = QtGui.QColor(50, 30, 0) - +QVARIANT_NULL = None QT_MAX_INT = 2147483647 -LOG_HIGHLIGHT_ERROR = [ - 'error', 'aborted', 'fatal', 'failed', 'killed', 'command not found', - 'no licenses could be found', 'killMessage'] -LOG_HIGHLIGHT_WARN = ['warning', 'not found'] -LOG_HIGHLIGHT_INFO = ['info:', 'rqd cmd:'] - - -LOG_ROOT_OS = { - "rhel7": "/shots", - "linux": "/shots", - "windows": "S:", - "mac": "/Users/shots" -} +COLUMN_INFO_DISPLAY = 2 diff --git a/cuegui/cuegui/Config.py b/cuegui/cuegui/Layout.py similarity index 94% rename from cuegui/cuegui/Config.py rename to cuegui/cuegui/Layout.py index 2c80f8492..65bb9c080 100644 --- a/cuegui/cuegui/Config.py +++ b/cuegui/cuegui/Layout.py @@ -13,7 +13,7 @@ # limitations under the License. -"""Functions for loading application state and settings from disk.""" +"""Functions for loading application layout and other state from disk.""" from __future__ import print_function from __future__ import division @@ -39,8 +39,7 @@ def startup(app_name): :return: settings object containing the loaded settings :rtype: QtCore.QSettings """ - # read saved config from disk - # copy default config + # E.g. ~/.config/.cuecommander/config.ini config_path = "/.%s/config" % app_name.lower() settings = QtCore.QSettings(QtCore.QSettings.IniFormat, QtCore.QSettings.UserScope, config_path) logger.info('Reading config file from %s', settings.fileName()) diff --git a/cuegui/cuegui/Main.py b/cuegui/cuegui/Main.py index 7712a4cdb..8e5299ef9 100644 --- a/cuegui/cuegui/Main.py +++ b/cuegui/cuegui/Main.py @@ -25,7 +25,7 @@ from PySide2 import QtGui import cuegui -import cuegui.Config +import cuegui.Layout import cuegui.Constants import cuegui.Logger import cuegui.MainWindow @@ -69,7 +69,7 @@ def startup(app_name, app_version, argv): app.threadpool = cuegui.ThreadPool.ThreadPool(3, parent=app) - settings = cuegui.Config.startup(app_name) + settings = cuegui.Layout.startup(app_name) app.settings = settings cuegui.Style.init() diff --git a/cuegui/cuegui/Redirect.py b/cuegui/cuegui/Redirect.py index ae46814f2..43406793c 100644 --- a/cuegui/cuegui/Redirect.py +++ b/cuegui/cuegui/Redirect.py @@ -543,7 +543,7 @@ def __isBurstSafe(self, alloc, procs, show): burst target show burst and the number of cores being redirected. If there's a number of cores that may not be possible to pick up by the target show, that number should be lower than the threshold set in the - cue_resources config. + cuegui.yaml `resources` config. @param alloc: The name of the allocation for the cores @type alloc: str diff --git a/cuegui/cuegui/Utils.py b/cuegui/cuegui/Utils.py index 07cb5c025..957f91908 100644 --- a/cuegui/cuegui/Utils.py +++ b/cuegui/cuegui/Utils.py @@ -36,8 +36,6 @@ from PySide2 import QtGui from PySide2 import QtWidgets import six -import yaml -from yaml.scanner import ScannerError import opencue import opencue.wrappers.group @@ -389,26 +387,16 @@ def memoryToString(kmem, unit=None): return "%.01fG" % (float(kmem) / pow(k, 2)) -def getResourceConfig(path=None): +def getResourceConfig(): """Reads the given yaml file and returns the entries as a dictionary. If no config path is given, the default resources config will be read If the given path does not exist, a warning will be printed and an empty dictionary will be returned - @param path: The path for the yaml file to read - @type path: str - @return: The entries in the given yaml file + @return: Resource config settings @rtype: dict """ - config = {} - if not path: - path = '{}/cue_resources.yaml'.format(cuegui.Constants.DEFAULT_INI_PATH) - try: - with open(path, 'r') as fileObject: - config = yaml.load(fileObject, Loader=yaml.SafeLoader) - except (IOError, ScannerError) as e: - print('WARNING: Could not read config file %s: %s' % (path, e)) - return config + return cuegui.Constants.RESOURCE_LIMITS ################################################################################ diff --git a/cuegui/cuegui/config/cue_resources.yaml b/cuegui/cuegui/config/cue_resources.yaml deleted file mode 100644 index 501b6aff4..000000000 --- a/cuegui/cuegui/config/cue_resources.yaml +++ /dev/null @@ -1,28 +0,0 @@ - - -# Host Specs: -# Use this section to set the max cores and max memory based on the available -# hardware. -# These values are used by: -# - layer-properties -# - redirect plugin -# - service properties -max_cores: 32 -max_memory: 128 - -max_gpus: 8 -max_gpu_memory: 128 - - -# Redirect Plugin maximum allowed core-hour cutoff. -# Users will not be able to search for procs with frames that have been -# already used more than this many core-hours: -max_proc_hour_cutoff: 30 - -# Redirect plugin wasted cores threshold: -# When redirecting, and the target show is at or very close to subscription -# burst, killing frames will free up cores that may not be picked up by the -# target job. The plugin will warn the user if the number of potentially lost -# cores is higher that this threshold. To disable this warning, set the -# threshold to -1 -redirect_wasted_cores_threshold: 100 diff --git a/cuegui/cuegui/config/cuegui.yaml b/cuegui/cuegui/config/cuegui.yaml new file mode 100644 index 000000000..38f492cb7 --- /dev/null +++ b/cuegui/cuegui/config/cuegui.yaml @@ -0,0 +1,112 @@ +# Default CueGUI config file + +logger.format: '%(levelname)-9s %(module)-10s %(message)s' +logger.level: 'WARNING' + +# Path for static resources like images/icons. +paths.resources: './images' +# Path for various config files. +paths.config: './config' +# Path for the default application layout .ini file. If users do not have a layout stored +# in their local filesystem, the layout stored here will be copied. This value can also +# be set via the CUEGUI_DEFAULT_INI_PATH environment variable. +paths.default_ini_path: './config' +# Paths for CueGUI plugins. +paths.plugins: ['./plugins'] + +# How often the UI will refresh its contents. All values in milliseconds. +refresh.job_update_delay: 10000 +refresh.layer_update_delay: 10000 +refresh.frame_update_delay: 10000 +refresh.host_update_delay: 20000 +refresh.after_action_update_delay: 1000 +refresh.min_update_interval: 5000 + +# Log roots used by various operating systems. Used for remapping paths so logs produced on +# one platform will be accessible locally. +render_logs.root: + windows: 'S:' + mac: '/Users/shots' + darwin: '/Users/shots' + linux: '/shots' + rhel7: '/shots' +# Substrings which, when found in render logs, will cause that line to be highlighted. +render_logs.highlight.error: [ + 'error', 'aborted', 'fatal', 'failed', 'killed', 'command not found', + 'no licenses could be found', 'killMessage'] +render_logs.highlight.warning: ['warning', 'not found'] +render_logs.highlight.info: ['info:', 'rqd cmd:'] + +# File should be stored in paths.config. +style.style_sheet: 'darkpalette.qss' +style.font.family: 'Luxi Sans' +style.font.size: 10 +style.color_theme: 'plastique' +# RGB values. +style.colors.background: [ + [50, 50, 100], + [100, 100, 50], + [0, 50, 0], + [50, 30, 0], +] +style.colors.frame_state: + DEAD: [255, 0, 0] + DEPEND: [160, 32, 240] + EATEN: [150, 0, 0] + RUNNING: [200, 200, 55] + SETUP: [160, 32, 240] + SUCCEEDED: [55, 200, 55] + WAITING: [135, 207, 235] + CHECKPOINT: [61, 98, 247] + +# Default editor to use for viewing log files. +editor.windows: 'notepad' +editor.mac: 'open -t' +editor.linux: 'gview -R -m -M -U {config_path}/gvimrc +' + +resources: + # The max cores and max memory based on the available hardware. + # These values are used by: + # - layer-properties + # - redirect plugin + # - service properties + max_cores: 32 + max_memory: 128 + max_gpus: 8 + max_gpu_memory: 128 + # Redirect Plugin maximum allowed core-hour cutoff. + # Users will not be able to search for procs with frames that have been + # already used more than this many core-hours: + max_proc_hour_cutoff: 30 + # Redirect plugin wasted cores threshold: + # When redirecting, and the target show is at or very close to subscription + # burst, killing frames will free up cores that may not be picked up by the + # target job. The plugin will warn the user if the number of potentially lost + # cores is higher that this threshold. To disable this warning, set the + # threshold to -1. + redirect_wasted_cores_threshold: 100 + +links.user_guide: 'https://www.opencue.io/docs/' +links.issue.create: 'https://github.com/AcademySoftwareFoundation/OpenCue/issues/new' +# Appended to `links.issue.create`. +links.issue.suggestion: '?labels=enhancement&template=enhancement.md' +# Appended to `links.issue.create`. +links.issue.bug: '?labels=bug&template=bug_report.md' + +# List of tags to be used when viewing or editing tags. +allowed_tags: ['general', 'desktop', 'playblast', 'util', 'preprocess', 'wan', 'cuda', 'splathw', + 'naiad', 'massive'] + +email.subject_prefix: 'cuemail: please check ' +email.body_prefix: 'Your PSTs request that you check ' +email.body_suffix: "\n\n" +email.domain: '' + +# Unix epoch timestamp. If the user last viewed the startup notice before this time, the +# notice will be shown. +startup_notice.date: 0 +# Notice message. +startup_notice.msg: '' + +# Memory usage above this level will be displayed in a different color. +memory_warning_level: 5242880 diff --git a/cuegui/tests/Constants_tests.py b/cuegui/tests/Constants_tests.py new file mode 100644 index 000000000..9466dcece --- /dev/null +++ b/cuegui/tests/Constants_tests.py @@ -0,0 +1,166 @@ +# Copyright Contributors to the OpenCue Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +"""Tests for cuegui.Constants""" + + +from __future__ import print_function +from __future__ import division +from __future__ import absolute_import + +import importlib +import os + +import mock +import pyfakefs.fake_filesystem_unittest +from PySide2 import QtGui + +import opencue +import cuegui.Constants + + +CONFIG_YAML = ''' +unused_setting: some value +version: 98.707.68 +refresh.job_update_delay: 30000 + +logger.level: INFO +''' + + +class ConstantsTests(pyfakefs.fake_filesystem_unittest.TestCase): + def setUp(self): + self.setUpPyfakefs() + self.fs.add_real_file( + os.path.join(os.path.dirname(cuegui.__file__), 'config', 'cuegui.yaml'), read_only=True) + if 'CUEGUI_CONFIG_FILE' in os.environ: + del os.environ['CUEGUI_CONFIG_FILE'] + + def test__should_load_user_config_from_env_var(self): + config_file_path = '/path/to/config.yaml' + self.fs.create_file(config_file_path, contents=CONFIG_YAML) + os.environ['CUEGUI_CONFIG_FILE'] = config_file_path + + result = importlib.reload(cuegui.Constants) + + self.assertEqual('98.707.68', result.VERSION) + self.assertEqual(30000, result.JOB_UPDATE_DELAY) + self.assertEqual(10000, result.LAYER_UPDATE_DELAY) + + @mock.patch('platform.system', new=mock.Mock(return_value='Linux')) + @mock.patch('os.path.expanduser', new=mock.Mock(return_value='/home/username')) + def test__should_load_user_config_from_user_profile(self): + config_file_path = '/home/username/.config/opencue/cuegui.yaml' + self.fs.create_file(config_file_path, contents=CONFIG_YAML) + + result = importlib.reload(cuegui.Constants) + + self.assertEqual('98.707.68', result.VERSION) + self.assertEqual(30000, result.JOB_UPDATE_DELAY) + self.assertEqual(10000, result.LAYER_UPDATE_DELAY) + + @mock.patch('platform.system', new=mock.Mock(return_value='Linux')) + def test__should_use_default_values(self): + result = importlib.reload(cuegui.Constants) + + self.assertNotEqual('98.707.68', result.VERSION) + self.assertEqual(0, result.STARTUP_NOTICE_DATE) + self.assertEqual('', result.STARTUP_NOTICE_MSG) + self.assertEqual(10000, result.JOB_UPDATE_DELAY) + self.assertEqual(10000, result.LAYER_UPDATE_DELAY) + self.assertEqual(10000, result.FRAME_UPDATE_DELAY) + self.assertEqual(20000, result.HOST_UPDATE_DELAY) + self.assertEqual(1000, result.AFTER_ACTION_UPDATE_DELAY) + self.assertEqual(5, result.MINIMUM_UPDATE_INTERVAL) + self.assertEqual('Luxi Sans', result.FONT_FAMILY) + self.assertEqual(10, result.FONT_SIZE) + self.assertEqual( + os.path.join(os.path.dirname(cuegui.__file__), 'images'), result.RESOURCE_PATH) + self.assertEqual( + os.path.join(os.path.dirname(cuegui.__file__), 'config'), result.CONFIG_PATH) + self.assertEqual( + os.path.join(os.path.dirname(cuegui.__file__), 'config'), result.DEFAULT_INI_PATH) + self.assertEqual( + [os.path.join(os.path.dirname(cuegui.__file__), 'plugins')], + result.DEFAULT_PLUGIN_PATHS) + self.assertEqual('%(levelname)-9s %(module)-10s %(message)s', result.LOGGER_FORMAT) + self.assertEqual('WARNING', result.LOGGER_LEVEL) + self.assertEqual('cuemail: please check ', result.EMAIL_SUBJECT_PREFIX) + self.assertEqual('Your PSTs request that you check ', result.EMAIL_BODY_PREFIX) + self.assertEqual('\n\n', result.EMAIL_BODY_SUFFIX) + self.assertEqual('', result.EMAIL_DOMAIN) + self.assertEqual( + 'https://github.com/AcademySoftwareFoundation/OpenCue/issues/new', + result.GITHUB_CREATE_ISSUE_URL) + self.assertEqual('https://www.opencue.io/docs/', result.URL_USERGUIDE) + self.assertEqual( + 'https://github.com/AcademySoftwareFoundation/OpenCue/issues/new' + '?labels=enhancement&template=enhancement.md', result.URL_SUGGESTION) + self.assertEqual( + 'https://github.com/AcademySoftwareFoundation/OpenCue/issues/new' + '?labels=bug&template=bug_report.md', result.URL_BUG) + self.assertEqual( + 'gview -R -m -M -U %s +' % os.path.join( + os.path.dirname(cuegui.__file__), 'config', 'gvimrc'), + result.DEFAULT_EDITOR) + self.assertEqual({ + 'rhel7': '/shots', + 'linux': '/shots', + 'windows': 'S:', + 'mac': '/Users/shots', + 'darwin': '/Users/shots', + }, result.LOG_ROOT_OS) + self.assertEqual(( + 'general', 'desktop', 'playblast', 'util', 'preprocess', 'wan', 'cuda', 'splathw', + 'naiad', 'massive'), result.ALLOWED_TAGS) + self.assertEqual( + os.path.join(os.path.dirname(cuegui.__file__), 'config', 'darkpalette.qss'), + result.DARK_STYLE_SHEET) + self.assertEqual('plastique', result.COLOR_THEME) + self.assertEqual(QtGui.QColor(50, 50, 100), result.COLOR_USER_1) + self.assertEqual(QtGui.QColor(100, 100, 50), result.COLOR_USER_2) + self.assertEqual(QtGui.QColor(0, 50, 0), result.COLOR_USER_3) + self.assertEqual(QtGui.QColor(50, 30, 00), result.COLOR_USER_4) + self.assertEqual({ + opencue.api.job_pb2.DEAD: QtGui.QColor(255, 0, 0), + opencue.api.job_pb2.DEPEND: QtGui.QColor(160, 32, 240), + opencue.api.job_pb2.EATEN: QtGui.QColor(150, 0, 0), + opencue.api.job_pb2.RUNNING: QtGui.QColor(200, 200, 55), + opencue.api.job_pb2.SETUP: QtGui.QColor(160, 32, 240), + opencue.api.job_pb2.SUCCEEDED: QtGui.QColor(55, 200, 55), + opencue.api.job_pb2.WAITING: QtGui.QColor(135, 207, 235), + opencue.api.job_pb2.CHECKPOINT: QtGui.QColor(61, 98, 247), + }, result.RGB_FRAME_STATE) + self.assertEqual(5242880, result.MEMORY_WARNING_LEVEL) + self.assertEqual( + ['error', 'aborted', 'fatal', 'failed', 'killed', 'command not found', + 'no licenses could be found', 'killMessage'], result.LOG_HIGHLIGHT_ERROR) + self.assertEqual(['warning', 'not found'], result.LOG_HIGHLIGHT_WARN) + self.assertEqual(['info:', 'rqd cmd:'], result.LOG_HIGHLIGHT_INFO) + self.assertEqual(2147483647, result.QT_MAX_INT) + self.assertEqual({ + 'max_cores': 32, + 'max_gpu_memory': 128, + 'max_gpus': 8, + 'max_memory': 128, + 'max_proc_hour_cutoff': 30, + 'redirect_wasted_cores_threshold': 100, + }, result.RESOURCE_LIMITS) + + @mock.patch('platform.system', new=mock.Mock(return_value='Darwin')) + def test__should_use_mac_editor(self): + result = importlib.reload(cuegui.Constants) + + self.assertEqual('open -t', result.DEFAULT_EDITOR) diff --git a/cuegui/tests/Config_tests.py b/cuegui/tests/Layout_tests.py similarity index 86% rename from cuegui/tests/Config_tests.py rename to cuegui/tests/Layout_tests.py index 61ab2c193..c291c3b1e 100644 --- a/cuegui/tests/Config_tests.py +++ b/cuegui/tests/Layout_tests.py @@ -13,7 +13,7 @@ # limitations under the License. -"""Tests for cuegui.Config""" +"""Tests for cuegui.Layout""" from __future__ import print_function @@ -27,7 +27,7 @@ from PySide2 import QtCore -import cuegui.Config +import cuegui.Layout CONFIG_INI = ''' @@ -50,7 +50,7 @@ ''' -class ConfigTests(unittest.TestCase): +class LayoutTests(unittest.TestCase): def setUp(self): self.config_dir = tempfile.mkdtemp() QtCore.QSettings.setPath( @@ -59,34 +59,34 @@ def setUp(self): def tearDown(self): shutil.rmtree(self.config_dir) - def test__should_load_user_config(self): + def test__should_load_user_layout(self): app_name = 'arbitraryapp' config_file_path = os.path.join(self.config_dir, '.%s' % app_name, 'config.ini') os.mkdir(os.path.dirname(config_file_path)) with open(config_file_path, 'w') as fp: fp.write(CONFIG_INI) - settings = cuegui.Config.startup(app_name) + settings = cuegui.Layout.startup(app_name) self.assertEqual('0.14', settings.value('Version')) self.assertEqual('true', settings.value('CueCommander/Open')) self.assertEqual('CustomWindowTitle', settings.value('CueCommander/Title')) self.assertEqual('arbitrary-value', settings.value('CueCommander/OtherAttr')) - def test__should_load_default_config(self): - settings = cuegui.Config.startup('CueCommander') + def test__should_load_default_layout(self): + settings = cuegui.Layout.startup('CueCommander') self.assertEqual('false', settings.value('CueCommander/Open')) self.assertEqual('CueCommander', settings.value('CueCommander/Title')) self.assertFalse(settings.value('CueCommander/OtherAttr', False)) - def test__should_restore_default_config(self): + def test__should_restore_default_layout(self): config_file_path = os.path.join(self.config_dir, '.cuecommander', 'config.ini') os.mkdir(os.path.dirname(config_file_path)) with open(config_file_path, 'w') as fp: fp.write(CONFIG_WITH_RESTORE_FLAG) - settings = cuegui.Config.startup('CueCommander') + settings = cuegui.Layout.startup('CueCommander') self.assertEqual('false', settings.value('CueCommander/Open')) self.assertEqual('CueCommander', settings.value('CueCommander/Title')) diff --git a/cuegui/tests/Utils_tests.py b/cuegui/tests/Utils_tests.py index dddf46423..e2f4a69b0 100644 --- a/cuegui/tests/Utils_tests.py +++ b/cuegui/tests/Utils_tests.py @@ -69,6 +69,18 @@ def test_shouldSwallowExceptionAndReturnNone(self): self.assertIsNone(cuegui.Utils.findJob(jobName)) + def test_shouldReturnResourceLimitsFromYaml(self): + result = cuegui.Utils.getResourceConfig() + + self.assertEqual({ + 'max_cores': 32, + 'max_gpu_memory': 128, + 'max_gpus': 8, + 'max_memory': 128, + 'max_proc_hour_cutoff': 30, + 'redirect_wasted_cores_threshold': 100, + }, result) + if __name__ == '__main__': unittest.main() From bb745588f5d010d30a11aaa0c3a99be531f12a40 Mon Sep 17 00:00:00 2001 From: romainf-ubi <117918548+romainf-ubi@users.noreply.github.com> Date: Thu, 26 Jan 2023 19:08:46 -0500 Subject: [PATCH 18/56] [rqd] Add new config option RQD_USE_PATH_ENV_VAR. (#1241) --- rqd/rqd/__main__.py | 21 ++++++++++++--------- rqd/rqd/rqconstants.py | 8 +++++++- rqd/rqd/rqmachine.py | 2 ++ 3 files changed, 21 insertions(+), 10 deletions(-) diff --git a/rqd/rqd/__main__.py b/rqd/rqd/__main__.py index 32e6dd38b..605f72f22 100755 --- a/rqd/rqd/__main__.py +++ b/rqd/rqd/__main__.py @@ -28,7 +28,7 @@ Optional configuration file: ---------------------------- -in /etc/rqd3/rqd3.conf: +In /etc/opencue/rqd.conf (on Linux) or %LOCALAPPDATA%/OpenCue/rqd.conf (on Windows): [Override] OVERRIDE_CORES = 2 OVERRIDE_PROCS = 3 @@ -89,14 +89,17 @@ def setupLogging(): def usage(): """Prints command line syntax""" - s = sys.stderr - print("SYNOPSIS", file=s) - print(" ", sys.argv[0], "[options]\n", file=s) - print(" -d | --daemon => Run as daemon", file=s) - print(" --nimbyoff => Disables nimby activation", file=s) - print(" -c => Provide an alternate config file", file=s) - print(" Defaults to /etc/rqd3/rqd3.conf", file=s) - print(" Config file is optional", file=s) + usage_msg = f"""SYNOPSIS + {sys.argv[0]} [options] + + -d | --daemon => Run as daemon + --nimbyoff => Disables nimby activation + -c => Provide an alternate config file + On Linux: defaults to /etc/opencue/rqd.conf + On Windows: Defaults to %LOCALAPPDATA%/OpenCue/rqd.conf + Config file is optional +""" + print(usage_msg, file=sys.stderr) def main(): diff --git a/rqd/rqd/rqconstants.py b/rqd/rqd/rqconstants.py index 0d6968d27..ad82b5daf 100644 --- a/rqd/rqd/rqconstants.py +++ b/rqd/rqd/rqconstants.py @@ -66,6 +66,10 @@ RQD_RETRY_CRITICAL_REPORT_DELAY = 30 RQD_USE_IP_AS_HOSTNAME = True RQD_USE_IPV6_AS_HOSTNAME = False + +# Use the PATH environment variable from the RQD host. +RQD_USE_PATH_ENV_VAR = False + RQD_BECOME_JOB_USER = True RQD_CREATE_USER_IF_NOT_EXISTS = True RQD_TAGS = '' @@ -111,7 +115,7 @@ SYS_HERTZ = os.sysconf('SC_CLK_TCK') if platform.system() == 'Windows': - CONFIG_FILE = os.path.expandvars('$LOCALAPPDATA/OpenCue/rqd.conf') + CONFIG_FILE = os.path.expandvars('%LOCALAPPDATA%/OpenCue/rqd.conf') else: CONFIG_FILE = '/etc/opencue/rqd.conf' @@ -177,6 +181,8 @@ RQD_USE_IP_AS_HOSTNAME = config.getboolean(__section, "RQD_USE_IP_AS_HOSTNAME") if config.has_option(__section, "RQD_USE_IPV6_AS_HOSTNAME"): RQD_USE_IPV6_AS_HOSTNAME = config.getboolean(__section, "RQD_USE_IPV6_AS_HOSTNAME") + if config.has_option(__section, "RQD_USE_PATH_ENV_VAR"): + RQD_USE_PATH_ENV_VAR = config.getboolean(__section, "RQD_USE_PATH_ENV_VAR") if config.has_option(__section, "RQD_BECOME_JOB_USER"): RQD_BECOME_JOB_USER = config.getboolean(__section, "RQD_BECOME_JOB_USER") if config.has_option(__section, "RQD_TAGS"): diff --git a/rqd/rqd/rqmachine.py b/rqd/rqd/rqmachine.py index 27295ed92..2fc5a32ce 100644 --- a/rqd/rqd/rqmachine.py +++ b/rqd/rqd/rqmachine.py @@ -507,6 +507,8 @@ def getHostname(self): @rqd.rqutil.Memoize def getPathEnv(self): """Returns the correct path environment for the given machine""" + if rqd.rqconstants.RQD_USE_PATH_ENV_VAR: + return os.getenv('PATH') if platform.system() == 'Linux': return '/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin' if platform.system() == 'Windows': From ce2253118b6808469c89aa54c227dd0e2df3171a Mon Sep 17 00:00:00 2001 From: Brian Cipriano Date: Mon, 6 Feb 2023 13:10:45 -0500 Subject: [PATCH 19/56] TSC meeting notes from Feb 1 meeting. (#1256) --- tsc/meetings/2023-02-01.md | 42 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) create mode 100644 tsc/meetings/2023-02-01.md diff --git a/tsc/meetings/2023-02-01.md b/tsc/meetings/2023-02-01.md new file mode 100644 index 000000000..88ef82f5b --- /dev/null +++ b/tsc/meetings/2023-02-01.md @@ -0,0 +1,42 @@ +# OpenCue TSC Meeting Notes 1 Feb 2023 + +Secretary: Brian Cipriano + +Agenda/Notes: + +* Move outline.cfg into outline module + * https://github.com/AcademySoftwareFoundation/OpenCue/pull/1252 + * What to do with bin/ and wrappers/ code? E.g. pycuerun. Packaged with rqd? + * What do these tools do? + * pycuerun is a tool for sending jobs to Cuebot. This should be packaged with pyoutline, + ideally as a console script / entrypoint. + * wrappers are used on the RQD side. These should probably be packaged with RQD somehow. + * Conclusion: we'll need different approaches for packaging these, needs some more research. +* Integration tests + * New PR to run a job and verify it + finishes: https://github.com/AcademySoftwareFoundation/OpenCue/pull/1253 + * Uses a pyoutline script to launch a job and a pycue script to wait for it to finish. + * Discovered outline.cfg issue above! + * Almost ready for review, waiting on pyoutline fix to be merged. +* Config guide + * CueGUI YAML config merged. + * PR with new config guide: https://github.com/AcademySoftwareFoundation/opencue.io/pull/274 + * Preview: + https://deploy-preview-274--elated-haibt-1b47ff.netlify.app/docs/other-guides/configuring-opencue/ +* PyPI + * Brian now doing some tests, cleaning up the pycue setup.py. + * Need to clean up dependency list in setup.py. This duplicates some work from requirements.txt, + but serves a different purpose, and the version restrictions should be a little looser. Have + to go one at a time to make a decision. +* RQD systemd changes + * https://github.com/AcademySoftwareFoundation/OpenCue/pull/1251 + * To be reviewed soon. + * Needs to be built into Docker image, CI pipelines. +* PySide6 + * https://github.com/AcademySoftwareFoundation/OpenCue/pull/1238 + * Review complete, merging soon. +* Migrate stats columns + * https://github.com/AcademySoftwareFoundation/OpenCue/pull/1228 + * Good way to roll out destructive changes? + * Any input from PG admins? + * Let's check with Diego next time, but current change looks fine. From 1c5dce48642c5c05f00e361ae9f947c221f21815 Mon Sep 17 00:00:00 2001 From: Brian Cipriano Date: Fri, 10 Feb 2023 12:46:50 -0500 Subject: [PATCH 20/56] [cuegui] Use the QtPy library instead of PySide directly. (#1238) --- .github/workflows/testing-pipeline.yml | 9 ++ VERSION.in | 2 +- ci/run_gui_test.sh | 8 +- ci/test_pyside6.sh | 36 ++++++++ cuegui/cuegui/AbstractDialog.py | 4 +- cuegui/cuegui/AbstractDockWidget.py | 4 +- cuegui/cuegui/AbstractTreeWidget.py | 6 +- cuegui/cuegui/AbstractWidgetItem.py | 4 +- cuegui/cuegui/Action.py | 4 +- cuegui/cuegui/App.py | 4 +- cuegui/cuegui/Comments.py | 4 +- cuegui/cuegui/ConfirmationDialog.py | 6 +- cuegui/cuegui/Constants.py | 4 +- cuegui/cuegui/CreateShowDialog.py | 4 +- cuegui/cuegui/CreatorDialog.py | 2 +- cuegui/cuegui/CueJobMonitorTree.py | 6 +- cuegui/cuegui/CueStateBarWidget.py | 6 +- cuegui/cuegui/DarkPalette.py | 4 +- cuegui/cuegui/DependDialog.py | 4 +- cuegui/cuegui/DependMonitorTree.py | 2 +- cuegui/cuegui/DependWizard.py | 4 +- cuegui/cuegui/EmailDialog.py | 6 +- cuegui/cuegui/FilterDialog.py | 8 +- cuegui/cuegui/FrameMonitor.py | 6 +- cuegui/cuegui/FrameMonitorTree.py | 6 +- cuegui/cuegui/FrameRangeSelection.py | 18 ++-- cuegui/cuegui/GarbageCollector.py | 2 +- cuegui/cuegui/GroupDialog.py | 4 +- cuegui/cuegui/HostMonitor.py | 4 +- cuegui/cuegui/HostMonitorTree.py | 6 +- cuegui/cuegui/ItemDelegate.py | 6 +- cuegui/cuegui/JobMonitorTree.py | 6 +- cuegui/cuegui/LayerDialog.py | 4 +- cuegui/cuegui/LayerMonitorTree.py | 4 +- cuegui/cuegui/Layout.py | 2 +- cuegui/cuegui/LimitSelectionWidget.py | 2 +- cuegui/cuegui/LimitsWidget.py | 4 +- cuegui/cuegui/LocalBooking.py | 4 +- cuegui/cuegui/Main.py | 2 +- cuegui/cuegui/MainWindow.py | 6 +- cuegui/cuegui/MenuActions.py | 4 +- cuegui/cuegui/MiscDialog.py | 2 +- cuegui/cuegui/Plugins.py | 4 +- cuegui/cuegui/PreviewWidget.py | 6 +- cuegui/cuegui/ProcChildren.py | 4 +- cuegui/cuegui/ProcMonitor.py | 4 +- cuegui/cuegui/ProcMonitorTree.py | 4 +- cuegui/cuegui/ProgressDialog.py | 4 +- cuegui/cuegui/Redirect.py | 6 +- cuegui/cuegui/ServiceDialog.py | 4 +- cuegui/cuegui/ShowDialog.py | 4 +- cuegui/cuegui/ShowsWidget.py | 4 +- cuegui/cuegui/SplashWindow.py | 6 +- cuegui/cuegui/Style.py | 2 +- cuegui/cuegui/SubscriptionGraphWidget.py | 4 +- cuegui/cuegui/SubscriptionsWidget.py | 4 +- cuegui/cuegui/TagsWidget.py | 2 +- cuegui/cuegui/TasksDialog.py | 4 +- cuegui/cuegui/TextEditDialog.py | 4 +- cuegui/cuegui/ThreadPool.py | 2 +- cuegui/cuegui/UnbookDialog.py | 4 +- cuegui/cuegui/Utils.py | 6 +- cuegui/cuegui/images/crystal/icons_rcc.py | 2 +- cuegui/cuegui/images/icons_rcc.py | 2 +- cuegui/cuegui/plugins/AllocationsPlugin.py | 2 +- cuegui/cuegui/plugins/AttributesPlugin.py | 4 +- cuegui/cuegui/plugins/LogViewPlugin.py | 8 +- cuegui/cuegui/plugins/MonitorCuePlugin.py | 6 +- cuegui/cuegui/plugins/MonitorHostsPlugin.py | 4 +- .../cuegui/plugins/MonitorJobDetailsPlugin.py | 4 +- cuegui/cuegui/plugins/MonitorJobsPlugin.py | 6 +- cuegui/cuegui/plugins/ShowsPlugin.py | 2 +- cuegui/cuegui/plugins/StuckFramePlugin.py | 6 +- .../plugins/SubscriptionsGraphPlugin.py | 4 +- cuegui/cuegui/plugins/SubscriptionsPlugin.py | 4 +- cuegui/setup.py | 1 + cuegui/tests/CueJobMonitorTree_tests.py | 10 +-- cuegui/tests/DependWizard_tests.py | 12 +-- cuegui/tests/FilterDialog_tests.py | 62 ++++++------- cuegui/tests/FrameMonitorTree_tests.py | 44 ++++----- cuegui/tests/LayerDialog_tests.py | 10 +-- cuegui/tests/Layout_tests.py | 2 +- cuegui/tests/MenuActions_tests.py | 90 +++++++++---------- cuegui/tests/Redirect_tests.py | 6 +- cuegui/tests/UnbookDialog_tests.py | 24 ++--- cuegui/tests/plugins/LogViewPlugin_tests.py | 28 +++--- requirements_gui.txt | 2 + 87 files changed, 357 insertions(+), 303 deletions(-) create mode 100755 ci/test_pyside6.sh diff --git a/.github/workflows/testing-pipeline.yml b/.github/workflows/testing-pipeline.yml index 98e5ca166..7d2b59af9 100644 --- a/.github/workflows/testing-pipeline.yml +++ b/.github/workflows/testing-pipeline.yml @@ -91,6 +91,15 @@ jobs: chown -R aswfuser:aswfgroup . su -c "cd cuebot && ./gradlew build --stacktrace --info" aswfuser + test_pyside6: + name: Run CueGUI Tests using PySide6 + runs-on: ubuntu-latest + container: almalinux:9 + steps: + - uses: actions/checkout@v2 + - name: Run CueGUI Tests + run: ci/test_pyside6.sh + lint_python: name: Lint Python Code runs-on: ubuntu-latest diff --git a/VERSION.in b/VERSION.in index 5320adc1c..e34629406 100644 --- a/VERSION.in +++ b/VERSION.in @@ -1 +1 @@ -0.21 +0.22 diff --git a/ci/run_gui_test.sh b/ci/run_gui_test.sh index b80505bdb..3c7d92a6d 100755 --- a/ci/run_gui_test.sh +++ b/ci/run_gui_test.sh @@ -14,8 +14,14 @@ # > OK # +py="$(command -v python3)" +if [[ -z "$py" ]]; then + py="$(command -v python)" +fi +echo "Using Python binary ${py}" + test_log="/tmp/cuegui_result.log" -PYTHONPATH=pycue xvfb-run -d python cuegui/setup.py test | tee ${test_log} +PYTHONPATH=pycue xvfb-run -d "${py}" cuegui/setup.py test | tee ${test_log} grep -Pz 'Ran \d+ tests in [0-9\.]+s\n\nOK' ${test_log} if [ $? -eq 0 ]; then diff --git a/ci/test_pyside6.sh b/ci/test_pyside6.sh new file mode 100755 index 000000000..05bd4c173 --- /dev/null +++ b/ci/test_pyside6.sh @@ -0,0 +1,36 @@ +#!/bin/bash + +# Script for testing CueGUI with PySide6. +# +# This script is written to be run within an almalinux environment in the OpenCue +# GitHub Actions environment. See .github/workflows/testing-pipeline.yml. + +set -e + +# Install needed packages. +yum -y install \ + dbus-libs \ + fontconfig \ + gcc \ + libxkbcommon-x11 \ + mesa-libEGL-devel \ + python-devel \ + which \ + xcb-util-keysyms \ + xcb-util-image \ + xcb-util-renderutil \ + xcb-util-wm \ + Xvfb + +# Install Python requirements. +python3 -m pip install --user -r requirements.txt -r requirements_gui.txt +# Replace PySide2 with PySide6. +python3 -m pip uninstall -y PySide2 +python3 -m pip install --user PySide6==6.3.2 + +# Fix compiled proto code for Python 3. +python3 -m grpc_tools.protoc -I=proto/ --python_out=pycue/opencue/compiled_proto --grpc_python_out=pycue/opencue/compiled_proto proto/*.proto +2to3 -wn -f import pycue/opencue/compiled_proto/*_pb2*.py + +# Run tests. +ci/run_gui_test.sh diff --git a/cuegui/cuegui/AbstractDialog.py b/cuegui/cuegui/AbstractDialog.py index f6d701fe4..2474cf14b 100644 --- a/cuegui/cuegui/AbstractDialog.py +++ b/cuegui/cuegui/AbstractDialog.py @@ -21,8 +21,8 @@ from __future__ import absolute_import from builtins import str -from PySide2 import QtCore -from PySide2 import QtWidgets +from qtpy import QtCore +from qtpy import QtWidgets class AbstractDialog(QtWidgets.QDialog): diff --git a/cuegui/cuegui/AbstractDockWidget.py b/cuegui/cuegui/AbstractDockWidget.py index 15857d5a2..0697662e3 100644 --- a/cuegui/cuegui/AbstractDockWidget.py +++ b/cuegui/cuegui/AbstractDockWidget.py @@ -22,8 +22,8 @@ from __future__ import print_function from __future__ import division -from PySide2 import QtCore -from PySide2 import QtWidgets +from qtpy import QtCore +from qtpy import QtWidgets import cuegui.Plugins diff --git a/cuegui/cuegui/AbstractTreeWidget.py b/cuegui/cuegui/AbstractTreeWidget.py index 88e98ad4f..0f4e9500a 100644 --- a/cuegui/cuegui/AbstractTreeWidget.py +++ b/cuegui/cuegui/AbstractTreeWidget.py @@ -26,9 +26,9 @@ from builtins import range import time -from PySide2 import QtCore -from PySide2 import QtGui -from PySide2 import QtWidgets +from qtpy import QtCore +from qtpy import QtGui +from qtpy import QtWidgets import cuegui.AbstractWidgetItem import cuegui.Constants diff --git a/cuegui/cuegui/AbstractWidgetItem.py b/cuegui/cuegui/AbstractWidgetItem.py index 301a3aa59..8a05699e8 100644 --- a/cuegui/cuegui/AbstractWidgetItem.py +++ b/cuegui/cuegui/AbstractWidgetItem.py @@ -24,8 +24,8 @@ from builtins import str -from PySide2 import QtCore -from PySide2 import QtWidgets +from qtpy import QtCore +from qtpy import QtWidgets import cuegui.Constants import cuegui.Logger diff --git a/cuegui/cuegui/Action.py b/cuegui/cuegui/Action.py index f9a083e09..0f24c4957 100644 --- a/cuegui/cuegui/Action.py +++ b/cuegui/cuegui/Action.py @@ -20,8 +20,8 @@ from __future__ import print_function from __future__ import division -from PySide2 import QtGui -from PySide2 import QtWidgets +from qtpy import QtGui +from qtpy import QtWidgets import cuegui.Constants diff --git a/cuegui/cuegui/App.py b/cuegui/cuegui/App.py index e161420fe..8e5409520 100644 --- a/cuegui/cuegui/App.py +++ b/cuegui/cuegui/App.py @@ -14,8 +14,8 @@ """Module for CueGUI's custom QApplication and associated helper functions.""" -from PySide2 import QtCore -from PySide2 import QtWidgets +from qtpy import QtCore +from qtpy import QtWidgets import cuegui.Exception diff --git a/cuegui/cuegui/Comments.py b/cuegui/cuegui/Comments.py index 628c1b740..c58c7bd79 100644 --- a/cuegui/cuegui/Comments.py +++ b/cuegui/cuegui/Comments.py @@ -23,8 +23,8 @@ from builtins import str import pickle -from PySide2 import QtCore -from PySide2 import QtWidgets +from qtpy import QtCore +from qtpy import QtWidgets import cuegui.Utils diff --git a/cuegui/cuegui/ConfirmationDialog.py b/cuegui/cuegui/ConfirmationDialog.py index 68f0f7acd..919a19563 100644 --- a/cuegui/cuegui/ConfirmationDialog.py +++ b/cuegui/cuegui/ConfirmationDialog.py @@ -20,9 +20,9 @@ from __future__ import division from __future__ import absolute_import -from PySide2 import QtCore -from PySide2 import QtGui -from PySide2 import QtWidgets +from qtpy import QtCore +from qtpy import QtGui +from qtpy import QtWidgets class ConfirmationDialog(QtWidgets.QDialog): diff --git a/cuegui/cuegui/Constants.py b/cuegui/cuegui/Constants.py index 4dcef529d..38f5dbafe 100644 --- a/cuegui/cuegui/Constants.py +++ b/cuegui/cuegui/Constants.py @@ -25,8 +25,8 @@ import os import platform -from PySide2 import QtGui -from PySide2 import QtWidgets +from qtpy import QtGui +from qtpy import QtWidgets import yaml import opencue diff --git a/cuegui/cuegui/CreateShowDialog.py b/cuegui/cuegui/CreateShowDialog.py index 148bd52d2..36e66fb49 100644 --- a/cuegui/cuegui/CreateShowDialog.py +++ b/cuegui/cuegui/CreateShowDialog.py @@ -21,8 +21,8 @@ from __future__ import division -from PySide2 import QtCore -from PySide2 import QtWidgets +from qtpy import QtCore +from qtpy import QtWidgets import opencue diff --git a/cuegui/cuegui/CreatorDialog.py b/cuegui/cuegui/CreatorDialog.py index a28d2f00b..5d2e5e450 100644 --- a/cuegui/cuegui/CreatorDialog.py +++ b/cuegui/cuegui/CreatorDialog.py @@ -23,7 +23,7 @@ from builtins import str from builtins import zip -from PySide2 import QtWidgets +from qtpy import QtWidgets import opencue diff --git a/cuegui/cuegui/CueJobMonitorTree.py b/cuegui/cuegui/CueJobMonitorTree.py index b7f32ebec..2305835b1 100644 --- a/cuegui/cuegui/CueJobMonitorTree.py +++ b/cuegui/cuegui/CueJobMonitorTree.py @@ -25,9 +25,9 @@ from collections import namedtuple import time -from PySide2 import QtCore -from PySide2 import QtGui -from PySide2 import QtWidgets +from qtpy import QtCore +from qtpy import QtGui +from qtpy import QtWidgets import opencue import opencue.compiled_proto.job_pb2 diff --git a/cuegui/cuegui/CueStateBarWidget.py b/cuegui/cuegui/CueStateBarWidget.py index c35622f17..26b7feb6f 100644 --- a/cuegui/cuegui/CueStateBarWidget.py +++ b/cuegui/cuegui/CueStateBarWidget.py @@ -24,9 +24,9 @@ import time import weakref -from PySide2 import QtCore -from PySide2 import QtGui -from PySide2 import QtWidgets +from qtpy import QtCore +from qtpy import QtGui +from qtpy import QtWidgets import cuegui.Logger diff --git a/cuegui/cuegui/DarkPalette.py b/cuegui/cuegui/DarkPalette.py index 22b89bf68..fed62996b 100644 --- a/cuegui/cuegui/DarkPalette.py +++ b/cuegui/cuegui/DarkPalette.py @@ -22,8 +22,8 @@ import platform -from PySide2 import QtGui -from PySide2 import QtWidgets +from qtpy import QtGui +from qtpy import QtWidgets import cuegui.Constants diff --git a/cuegui/cuegui/DependDialog.py b/cuegui/cuegui/DependDialog.py index ce94f526f..613e3da39 100644 --- a/cuegui/cuegui/DependDialog.py +++ b/cuegui/cuegui/DependDialog.py @@ -20,8 +20,8 @@ from __future__ import print_function from __future__ import division -from PySide2 import QtCore -from PySide2 import QtWidgets +from qtpy import QtCore +from qtpy import QtWidgets import cuegui.DependMonitorTree import cuegui.Logger diff --git a/cuegui/cuegui/DependMonitorTree.py b/cuegui/cuegui/DependMonitorTree.py index fed246a36..8f778280b 100644 --- a/cuegui/cuegui/DependMonitorTree.py +++ b/cuegui/cuegui/DependMonitorTree.py @@ -22,7 +22,7 @@ from builtins import map -from PySide2 import QtWidgets +from qtpy import QtWidgets from opencue.compiled_proto import depend_pb2 import opencue.exception diff --git a/cuegui/cuegui/DependWizard.py b/cuegui/cuegui/DependWizard.py index f328f76d7..b07a5ee46 100644 --- a/cuegui/cuegui/DependWizard.py +++ b/cuegui/cuegui/DependWizard.py @@ -25,8 +25,8 @@ from builtins import range import re -from PySide2 import QtCore -from PySide2 import QtWidgets +from qtpy import QtCore +from qtpy import QtWidgets import FileSequence import opencue diff --git a/cuegui/cuegui/EmailDialog.py b/cuegui/cuegui/EmailDialog.py index f4f83387e..227b12627 100644 --- a/cuegui/cuegui/EmailDialog.py +++ b/cuegui/cuegui/EmailDialog.py @@ -38,9 +38,9 @@ pass import smtplib -from PySide2 import QtCore -from PySide2 import QtGui -from PySide2 import QtWidgets +from qtpy import QtCore +from qtpy import QtGui +from qtpy import QtWidgets import opencue diff --git a/cuegui/cuegui/FilterDialog.py b/cuegui/cuegui/FilterDialog.py index 46126050b..479f1c6e5 100644 --- a/cuegui/cuegui/FilterDialog.py +++ b/cuegui/cuegui/FilterDialog.py @@ -23,9 +23,9 @@ from builtins import map from builtins import str -from PySide2 import QtCore -from PySide2 import QtGui -from PySide2 import QtWidgets +from qtpy import QtCore +from qtpy import QtGui +from qtpy import QtWidgets import opencue import opencue.compiled_proto.filter_pb2 @@ -63,7 +63,7 @@ def __init__(self, show, parent=None): :type show: opencue.wrappers.show.Show :param show: the show to manage filters for - :type parent: PySide2.QtWidgets.QWidget.QWidget + :type parent: qtpy.QtWidgets.QWidget.QWidget :param parent: the parent widget """ QtWidgets.QDialog.__init__(self, parent) diff --git a/cuegui/cuegui/FrameMonitor.py b/cuegui/cuegui/FrameMonitor.py index a854d574f..56c586cd9 100644 --- a/cuegui/cuegui/FrameMonitor.py +++ b/cuegui/cuegui/FrameMonitor.py @@ -24,9 +24,9 @@ from copy import deepcopy import math -from PySide2 import QtCore -from PySide2 import QtGui -from PySide2 import QtWidgets +from qtpy import QtCore +from qtpy import QtGui +from qtpy import QtWidgets import FileSequence from opencue.compiled_proto import job_pb2 diff --git a/cuegui/cuegui/FrameMonitorTree.py b/cuegui/cuegui/FrameMonitorTree.py index b546be2d5..e40be0f4d 100644 --- a/cuegui/cuegui/FrameMonitorTree.py +++ b/cuegui/cuegui/FrameMonitorTree.py @@ -29,9 +29,9 @@ import re import time -from PySide2 import QtCore -from PySide2 import QtGui -from PySide2 import QtWidgets +from qtpy import QtCore +from qtpy import QtGui +from qtpy import QtWidgets import opencue from opencue.compiled_proto import job_pb2 diff --git a/cuegui/cuegui/FrameRangeSelection.py b/cuegui/cuegui/FrameRangeSelection.py index 851e2aab2..003c4bc22 100644 --- a/cuegui/cuegui/FrameRangeSelection.py +++ b/cuegui/cuegui/FrameRangeSelection.py @@ -25,9 +25,9 @@ from builtins import range import math -from PySide2 import QtCore -from PySide2 import QtGui -from PySide2 import QtWidgets +from qtpy import QtCore +from qtpy import QtGui +from qtpy import QtWidgets class FrameRangeSelectionWidget(QtWidgets.QWidget): @@ -303,7 +303,7 @@ def __paintLabels(self, painter): oldPen = painter.pen() # draw hatches for labelled frames - painter.setPen(self.palette().color(QtGui.QPalette.Foreground)) + painter.setPen(self.palette().color(QtGui.QPalette.WindowText)) for frame in frames: xPos = self.__getTickArea(frame).left() painter.drawLine(xPos, -labelHeight, xPos, 0) @@ -313,7 +313,7 @@ def __paintLabels(self, painter): metric = QtGui.QFontMetrics(painter.font()) yPos = metric.ascent() + 1 rightEdge = -10000 - width = metric.width(str(frames[-1])) + width = metric.horizontalAdvance(str(frames[-1])) farEdge = self.__getTickArea(frames[-1]).right() - width // 2 farEdge -= 4 @@ -321,7 +321,7 @@ def __paintLabels(self, painter): for frame in frames: xPos = self.__getTickArea(frame).left() frameString = str(frame) - width = metric.width(frameString) + width = metric.horizontalAdvance(frameString) xPos = xPos - width // 2 if (xPos > rightEdge and xPos + width < farEdge) or frame is frames[-1]: painter.drawText(xPos, yPos, frameString) @@ -337,7 +337,7 @@ def __paintStartTime(self, painter): metric = QtGui.QFontMetrics(painter.font()) frameString = str(int(startFrame)) - xPos = timeExtent.left() - metric.width(frameString) // 2 + xPos = timeExtent.left() - metric.horizontalAdvance(frameString) // 2 yPos = metric.ascent() + 1 painter.drawText(xPos, yPos, frameString) painter.setPen(oldPen) @@ -351,7 +351,7 @@ def __paintEndTime(self, painter): metric = QtGui.QFontMetrics(painter.font()) frameString = str(int(endFrame)) - xPos = timeExtent.left() - metric.width(frameString) // 2 + xPos = timeExtent.left() - metric.horizontalAdvance(frameString) // 2 yPos = metric.ascent() + 1 painter.drawText(xPos, yPos, frameString) painter.setPen(oldPen) @@ -372,7 +372,7 @@ def __paintFloatTime(self, painter): painter.setPen(QtGui.QColor(128, 128, 128)) metric = QtGui.QFontMetrics(painter.font()) frameString = str(self.__floatTime) - xPos = timeExtent.left() - metric.width(frameString) // 2 + xPos = timeExtent.left() - metric.horizontalAdvance(frameString) // 2 yPos = timeExtent.top() + metric.ascent() painter.drawText(xPos, yPos, frameString) painter.setPen(oldPen) diff --git a/cuegui/cuegui/GarbageCollector.py b/cuegui/cuegui/GarbageCollector.py index 83cc06bc7..7f999d98f 100644 --- a/cuegui/cuegui/GarbageCollector.py +++ b/cuegui/cuegui/GarbageCollector.py @@ -23,7 +23,7 @@ import gc -from PySide2 import QtCore +from qtpy import QtCore class GarbageCollector(QtCore.QObject): diff --git a/cuegui/cuegui/GroupDialog.py b/cuegui/cuegui/GroupDialog.py index 0f949b2f7..d5f26ac18 100644 --- a/cuegui/cuegui/GroupDialog.py +++ b/cuegui/cuegui/GroupDialog.py @@ -23,8 +23,8 @@ from builtins import str -from PySide2 import QtCore -from PySide2 import QtWidgets +from qtpy import QtCore +from qtpy import QtWidgets import opencue diff --git a/cuegui/cuegui/HostMonitor.py b/cuegui/cuegui/HostMonitor.py index cd7ffee99..66ee94287 100644 --- a/cuegui/cuegui/HostMonitor.py +++ b/cuegui/cuegui/HostMonitor.py @@ -22,8 +22,8 @@ from builtins import str -from PySide2 import QtCore -from PySide2 import QtWidgets +from qtpy import QtCore +from qtpy import QtWidgets import opencue diff --git a/cuegui/cuegui/HostMonitorTree.py b/cuegui/cuegui/HostMonitorTree.py index 1c247e2c6..bff3d6ab2 100644 --- a/cuegui/cuegui/HostMonitorTree.py +++ b/cuegui/cuegui/HostMonitorTree.py @@ -23,9 +23,9 @@ from builtins import map import time -from PySide2 import QtCore -from PySide2 import QtGui -from PySide2 import QtWidgets +from qtpy import QtCore +from qtpy import QtGui +from qtpy import QtWidgets import opencue from opencue.compiled_proto.host_pb2 import HardwareState diff --git a/cuegui/cuegui/ItemDelegate.py b/cuegui/cuegui/ItemDelegate.py index 27aff67ec..e4de3df02 100644 --- a/cuegui/cuegui/ItemDelegate.py +++ b/cuegui/cuegui/ItemDelegate.py @@ -24,9 +24,9 @@ from builtins import range from math import ceil -from PySide2 import QtCore -from PySide2 import QtGui -from PySide2 import QtWidgets +from qtpy import QtCore +from qtpy import QtGui +from qtpy import QtWidgets import opencue diff --git a/cuegui/cuegui/JobMonitorTree.py b/cuegui/cuegui/JobMonitorTree.py index 3bf15373b..d92c44afc 100644 --- a/cuegui/cuegui/JobMonitorTree.py +++ b/cuegui/cuegui/JobMonitorTree.py @@ -24,9 +24,9 @@ from builtins import map import time -from PySide2 import QtCore -from PySide2 import QtGui -from PySide2 import QtWidgets +from qtpy import QtCore +from qtpy import QtGui +from qtpy import QtWidgets import opencue diff --git a/cuegui/cuegui/LayerDialog.py b/cuegui/cuegui/LayerDialog.py index 5f25be5f6..8c9f0e57f 100644 --- a/cuegui/cuegui/LayerDialog.py +++ b/cuegui/cuegui/LayerDialog.py @@ -20,8 +20,8 @@ from __future__ import division from __future__ import print_function -from PySide2 import QtCore -from PySide2 import QtWidgets +from qtpy import QtCore +from qtpy import QtWidgets import opencue diff --git a/cuegui/cuegui/LayerMonitorTree.py b/cuegui/cuegui/LayerMonitorTree.py index 7663d3276..25916e16f 100644 --- a/cuegui/cuegui/LayerMonitorTree.py +++ b/cuegui/cuegui/LayerMonitorTree.py @@ -20,8 +20,8 @@ from __future__ import print_function from __future__ import division -from PySide2 import QtCore -from PySide2 import QtWidgets +from qtpy import QtCore +from qtpy import QtWidgets from opencue.exception import EntityNotFoundException diff --git a/cuegui/cuegui/Layout.py b/cuegui/cuegui/Layout.py index 65bb9c080..9b26d23c8 100644 --- a/cuegui/cuegui/Layout.py +++ b/cuegui/cuegui/Layout.py @@ -22,7 +22,7 @@ import os import shutil -from PySide2 import QtCore +from qtpy import QtCore import cuegui.Constants import cuegui.Logger diff --git a/cuegui/cuegui/LimitSelectionWidget.py b/cuegui/cuegui/LimitSelectionWidget.py index 8a37e370e..a760d1b79 100644 --- a/cuegui/cuegui/LimitSelectionWidget.py +++ b/cuegui/cuegui/LimitSelectionWidget.py @@ -22,7 +22,7 @@ from builtins import str -from PySide2 import QtWidgets +from qtpy import QtWidgets import cuegui.AbstractDialog diff --git a/cuegui/cuegui/LimitsWidget.py b/cuegui/cuegui/LimitsWidget.py index 26f871de1..25c962492 100644 --- a/cuegui/cuegui/LimitsWidget.py +++ b/cuegui/cuegui/LimitsWidget.py @@ -20,8 +20,8 @@ from __future__ import print_function from __future__ import division -from PySide2 import QtCore -from PySide2 import QtWidgets +from qtpy import QtCore +from qtpy import QtWidgets import opencue diff --git a/cuegui/cuegui/LocalBooking.py b/cuegui/cuegui/LocalBooking.py index a69270198..23220e113 100644 --- a/cuegui/cuegui/LocalBooking.py +++ b/cuegui/cuegui/LocalBooking.py @@ -27,8 +27,8 @@ import os from socket import gethostname -from PySide2 import QtCore -from PySide2 import QtWidgets +from qtpy import QtCore +from qtpy import QtWidgets import opencue diff --git a/cuegui/cuegui/Main.py b/cuegui/cuegui/Main.py index 8e5299ef9..24f08864b 100644 --- a/cuegui/cuegui/Main.py +++ b/cuegui/cuegui/Main.py @@ -22,7 +22,7 @@ import signal -from PySide2 import QtGui +from qtpy import QtGui import cuegui import cuegui.Layout diff --git a/cuegui/cuegui/MainWindow.py b/cuegui/cuegui/MainWindow.py index 50555f647..942a9a717 100644 --- a/cuegui/cuegui/MainWindow.py +++ b/cuegui/cuegui/MainWindow.py @@ -28,9 +28,9 @@ import sys import time -from PySide2 import QtCore -from PySide2 import QtGui -from PySide2 import QtWidgets +from qtpy import QtCore +from qtpy import QtGui +from qtpy import QtWidgets import opencue diff --git a/cuegui/cuegui/MenuActions.py b/cuegui/cuegui/MenuActions.py index 2f6771baa..8dc89577f 100644 --- a/cuegui/cuegui/MenuActions.py +++ b/cuegui/cuegui/MenuActions.py @@ -28,8 +28,8 @@ import subprocess import time -from PySide2 import QtGui -from PySide2 import QtWidgets +from qtpy import QtGui +from qtpy import QtWidgets import six import FileSequence diff --git a/cuegui/cuegui/MiscDialog.py b/cuegui/cuegui/MiscDialog.py index c3b1185f8..d2712af93 100644 --- a/cuegui/cuegui/MiscDialog.py +++ b/cuegui/cuegui/MiscDialog.py @@ -20,7 +20,7 @@ from __future__ import print_function from __future__ import division -from PySide2 import QtWidgets +from qtpy import QtWidgets import cuegui.AbstractDialog diff --git a/cuegui/cuegui/Plugins.py b/cuegui/cuegui/Plugins.py index f4e6e3524..a4d48563c 100644 --- a/cuegui/cuegui/Plugins.py +++ b/cuegui/cuegui/Plugins.py @@ -57,8 +57,8 @@ import traceback import pickle -from PySide2 import QtCore -from PySide2 import QtWidgets +from qtpy import QtCore +from qtpy import QtWidgets import cuegui.Constants import cuegui.Logger diff --git a/cuegui/cuegui/PreviewWidget.py b/cuegui/cuegui/PreviewWidget.py index 56888188e..5dfea8658 100644 --- a/cuegui/cuegui/PreviewWidget.py +++ b/cuegui/cuegui/PreviewWidget.py @@ -33,8 +33,8 @@ import urllib.request import xml.etree.ElementTree as Et -from PySide2 import QtCore -from PySide2 import QtWidgets +from qtpy import QtCore +from qtpy import QtWidgets import cuegui.Logger import cuegui.Utils @@ -54,7 +54,7 @@ def __init__(self, job, frame, aovs=False, parent=None): :param frame: frame to display :type aovs: bool :param aovs: whether to display AOVs or just the main image - :type parent: PySide2.QtWidgets.QWidget + :type parent: qtpy.QtWidgets.QWidget :param parent: the parent widget """ QtWidgets.QDialog.__init__(self, parent) diff --git a/cuegui/cuegui/ProcChildren.py b/cuegui/cuegui/ProcChildren.py index 5faa0415e..c281f7970 100644 --- a/cuegui/cuegui/ProcChildren.py +++ b/cuegui/cuegui/ProcChildren.py @@ -27,8 +27,8 @@ from builtins import str -from PySide2 import QtGui -from PySide2 import QtWidgets +from qtpy import QtGui +from qtpy import QtWidgets import opencue diff --git a/cuegui/cuegui/ProcMonitor.py b/cuegui/cuegui/ProcMonitor.py index 4221a2c92..b4ae4f5d4 100644 --- a/cuegui/cuegui/ProcMonitor.py +++ b/cuegui/cuegui/ProcMonitor.py @@ -22,8 +22,8 @@ from builtins import str -from PySide2 import QtCore -from PySide2 import QtWidgets +from qtpy import QtCore +from qtpy import QtWidgets import cuegui.Logger import cuegui.ProcMonitorTree diff --git a/cuegui/cuegui/ProcMonitorTree.py b/cuegui/cuegui/ProcMonitorTree.py index cf2658de5..0f4710b53 100644 --- a/cuegui/cuegui/ProcMonitorTree.py +++ b/cuegui/cuegui/ProcMonitorTree.py @@ -23,8 +23,8 @@ from builtins import map import time -from PySide2 import QtCore -from PySide2 import QtWidgets +from qtpy import QtCore +from qtpy import QtWidgets import opencue diff --git a/cuegui/cuegui/ProgressDialog.py b/cuegui/cuegui/ProgressDialog.py index 2f507fda7..f9b49c453 100644 --- a/cuegui/cuegui/ProgressDialog.py +++ b/cuegui/cuegui/ProgressDialog.py @@ -23,8 +23,8 @@ from builtins import map from builtins import range -from PySide2 import QtCore -from PySide2 import QtWidgets +from qtpy import QtCore +from qtpy import QtWidgets import cuegui.Logger import cuegui.Utils diff --git a/cuegui/cuegui/Redirect.py b/cuegui/cuegui/Redirect.py index 43406793c..191bfb6ad 100644 --- a/cuegui/cuegui/Redirect.py +++ b/cuegui/cuegui/Redirect.py @@ -31,9 +31,9 @@ import re import time -from PySide2 import QtCore -from PySide2 import QtGui -from PySide2 import QtWidgets +from qtpy import QtCore +from qtpy import QtGui +from qtpy import QtWidgets import opencue diff --git a/cuegui/cuegui/ServiceDialog.py b/cuegui/cuegui/ServiceDialog.py index b29a13c23..c54ca4a88 100644 --- a/cuegui/cuegui/ServiceDialog.py +++ b/cuegui/cuegui/ServiceDialog.py @@ -23,8 +23,8 @@ from builtins import str from builtins import range -from PySide2 import QtCore -from PySide2 import QtWidgets +from qtpy import QtCore +from qtpy import QtWidgets import opencue diff --git a/cuegui/cuegui/ShowDialog.py b/cuegui/cuegui/ShowDialog.py index 7acdfec26..f0376fcf5 100644 --- a/cuegui/cuegui/ShowDialog.py +++ b/cuegui/cuegui/ShowDialog.py @@ -22,8 +22,8 @@ from builtins import str -from PySide2 import QtCore -from PySide2 import QtWidgets +from qtpy import QtCore +from qtpy import QtWidgets import cuegui.Utils diff --git a/cuegui/cuegui/ShowsWidget.py b/cuegui/cuegui/ShowsWidget.py index f10fa08e3..20b62ae73 100644 --- a/cuegui/cuegui/ShowsWidget.py +++ b/cuegui/cuegui/ShowsWidget.py @@ -20,8 +20,8 @@ from __future__ import print_function from __future__ import division -from PySide2 import QtCore -from PySide2 import QtWidgets +from qtpy import QtCore +from qtpy import QtWidgets import opencue diff --git a/cuegui/cuegui/SplashWindow.py b/cuegui/cuegui/SplashWindow.py index 29c268c7a..57075a261 100644 --- a/cuegui/cuegui/SplashWindow.py +++ b/cuegui/cuegui/SplashWindow.py @@ -24,9 +24,9 @@ import os import time -from PySide2 import QtCore -from PySide2 import QtGui -from PySide2 import QtWidgets +from qtpy import QtCore +from qtpy import QtGui +from qtpy import QtWidgets __all__ = ["SplashWindow"] diff --git a/cuegui/cuegui/Style.py b/cuegui/cuegui/Style.py index d95b80dd8..95acea64f 100644 --- a/cuegui/cuegui/Style.py +++ b/cuegui/cuegui/Style.py @@ -22,7 +22,7 @@ import importlib -from PySide2 import QtGui +from qtpy import QtGui import cuegui diff --git a/cuegui/cuegui/SubscriptionGraphWidget.py b/cuegui/cuegui/SubscriptionGraphWidget.py index e129ed3eb..b17004d11 100644 --- a/cuegui/cuegui/SubscriptionGraphWidget.py +++ b/cuegui/cuegui/SubscriptionGraphWidget.py @@ -20,8 +20,8 @@ import opencue -from PySide2 import QtCore -from PySide2 import QtWidgets +from qtpy import QtCore +from qtpy import QtWidgets import cuegui.AbstractTreeWidget import cuegui.AbstractWidgetItem diff --git a/cuegui/cuegui/SubscriptionsWidget.py b/cuegui/cuegui/SubscriptionsWidget.py index 575742f4b..eb474e074 100644 --- a/cuegui/cuegui/SubscriptionsWidget.py +++ b/cuegui/cuegui/SubscriptionsWidget.py @@ -24,8 +24,8 @@ import opencue -from PySide2 import QtCore -from PySide2 import QtWidgets +from qtpy import QtCore +from qtpy import QtWidgets import cuegui.AbstractTreeWidget import cuegui.AbstractWidgetItem diff --git a/cuegui/cuegui/TagsWidget.py b/cuegui/cuegui/TagsWidget.py index e9d6dc129..548ba2b53 100644 --- a/cuegui/cuegui/TagsWidget.py +++ b/cuegui/cuegui/TagsWidget.py @@ -23,7 +23,7 @@ from builtins import str import re -from PySide2 import QtWidgets +from qtpy import QtWidgets import cuegui.AbstractDialog import cuegui.Constants diff --git a/cuegui/cuegui/TasksDialog.py b/cuegui/cuegui/TasksDialog.py index 614b07c2d..2f820d76c 100644 --- a/cuegui/cuegui/TasksDialog.py +++ b/cuegui/cuegui/TasksDialog.py @@ -23,8 +23,8 @@ from builtins import map from builtins import str -from PySide2 import QtCore -from PySide2 import QtWidgets +from qtpy import QtCore +from qtpy import QtWidgets import opencue.exception diff --git a/cuegui/cuegui/TextEditDialog.py b/cuegui/cuegui/TextEditDialog.py index ed25fe45d..a45687d92 100644 --- a/cuegui/cuegui/TextEditDialog.py +++ b/cuegui/cuegui/TextEditDialog.py @@ -20,8 +20,8 @@ from __future__ import division from __future__ import absolute_import -from PySide2 import QtCore -from PySide2 import QtWidgets +from qtpy import QtCore +from qtpy import QtWidgets class TextEditDialog(QtWidgets.QDialog): diff --git a/cuegui/cuegui/ThreadPool.py b/cuegui/cuegui/ThreadPool.py index ba03ddc8f..fa6e6c412 100644 --- a/cuegui/cuegui/ThreadPool.py +++ b/cuegui/cuegui/ThreadPool.py @@ -49,7 +49,7 @@ def someWorkCallback(work, result): from builtins import range import os -from PySide2 import QtCore +from qtpy import QtCore import cuegui.Logger diff --git a/cuegui/cuegui/UnbookDialog.py b/cuegui/cuegui/UnbookDialog.py index e74149665..a661faa93 100644 --- a/cuegui/cuegui/UnbookDialog.py +++ b/cuegui/cuegui/UnbookDialog.py @@ -25,8 +25,8 @@ from builtins import object import re -from PySide2 import QtCore -from PySide2 import QtWidgets +from qtpy import QtCore +from qtpy import QtWidgets import six import opencue diff --git a/cuegui/cuegui/Utils.py b/cuegui/cuegui/Utils.py index 957f91908..4666797b5 100644 --- a/cuegui/cuegui/Utils.py +++ b/cuegui/cuegui/Utils.py @@ -32,9 +32,9 @@ import traceback import webbrowser -from PySide2 import QtCore -from PySide2 import QtGui -from PySide2 import QtWidgets +from qtpy import QtCore +from qtpy import QtGui +from qtpy import QtWidgets import six import opencue diff --git a/cuegui/cuegui/images/crystal/icons_rcc.py b/cuegui/cuegui/images/crystal/icons_rcc.py index 87d4c83d9..d66df7306 100644 --- a/cuegui/cuegui/images/crystal/icons_rcc.py +++ b/cuegui/cuegui/images/crystal/icons_rcc.py @@ -23,7 +23,7 @@ # # WARNING! All changes made in this file will be lost! -from PySide2 import QtCore +from qtpy import QtCore qt_resource_data = b"\ \x00\x00\x02\xfc\ diff --git a/cuegui/cuegui/images/icons_rcc.py b/cuegui/cuegui/images/icons_rcc.py index b313c2d56..4a9664ee7 100644 --- a/cuegui/cuegui/images/icons_rcc.py +++ b/cuegui/cuegui/images/icons_rcc.py @@ -28,7 +28,7 @@ from __future__ import division from __future__ import absolute_import -from PySide2 import QtCore +from qtpy import QtCore qt_resource_data = b"\ diff --git a/cuegui/cuegui/plugins/AllocationsPlugin.py b/cuegui/cuegui/plugins/AllocationsPlugin.py index 9e7d1a647..a7d18162e 100644 --- a/cuegui/cuegui/plugins/AllocationsPlugin.py +++ b/cuegui/cuegui/plugins/AllocationsPlugin.py @@ -22,7 +22,7 @@ from builtins import map -from PySide2 import QtWidgets +from qtpy import QtWidgets import opencue diff --git a/cuegui/cuegui/plugins/AttributesPlugin.py b/cuegui/cuegui/plugins/AttributesPlugin.py index 26f9350de..c25480f85 100644 --- a/cuegui/cuegui/plugins/AttributesPlugin.py +++ b/cuegui/cuegui/plugins/AttributesPlugin.py @@ -24,8 +24,8 @@ from builtins import str import time -from PySide2 import QtCore -from PySide2 import QtWidgets +from qtpy import QtCore +from qtpy import QtWidgets import opencue import opencue.compiled_proto.depend_pb2 diff --git a/cuegui/cuegui/plugins/LogViewPlugin.py b/cuegui/cuegui/plugins/LogViewPlugin.py index caeacf449..1e67ac1c4 100644 --- a/cuegui/cuegui/plugins/LogViewPlugin.py +++ b/cuegui/cuegui/plugins/LogViewPlugin.py @@ -27,9 +27,9 @@ import string import time -from PySide2 import QtGui -from PySide2 import QtCore -from PySide2 import QtWidgets +from qtpy import QtGui +from qtpy import QtCore +from qtpy import QtWidgets import cuegui.Constants import cuegui.AbstractDockWidget @@ -183,7 +183,7 @@ def get_line_number_area_width(self): while count >= 10: count /= 10 digits += 1 - space = 3 + self.fontMetrics().width('9') * digits + space = 3 + self.fontMetrics().horizontalAdvance('9') * digits return space def update_line_number_area_width(self): diff --git a/cuegui/cuegui/plugins/MonitorCuePlugin.py b/cuegui/cuegui/plugins/MonitorCuePlugin.py index c0b3e6fac..7884bfc5a 100644 --- a/cuegui/cuegui/plugins/MonitorCuePlugin.py +++ b/cuegui/cuegui/plugins/MonitorCuePlugin.py @@ -25,9 +25,9 @@ import re import weakref -from PySide2 import QtGui -from PySide2 import QtCore -from PySide2 import QtWidgets +from qtpy import QtGui +from qtpy import QtCore +from qtpy import QtWidgets import opencue diff --git a/cuegui/cuegui/plugins/MonitorHostsPlugin.py b/cuegui/cuegui/plugins/MonitorHostsPlugin.py index cee8413fe..be40d7453 100644 --- a/cuegui/cuegui/plugins/MonitorHostsPlugin.py +++ b/cuegui/cuegui/plugins/MonitorHostsPlugin.py @@ -20,8 +20,8 @@ from __future__ import division from __future__ import absolute_import -from PySide2 import QtCore -from PySide2 import QtWidgets +from qtpy import QtCore +from qtpy import QtWidgets import cuegui.AbstractDockWidget import cuegui.HostMonitor diff --git a/cuegui/cuegui/plugins/MonitorJobDetailsPlugin.py b/cuegui/cuegui/plugins/MonitorJobDetailsPlugin.py index 270a74b23..bb1b61d02 100644 --- a/cuegui/cuegui/plugins/MonitorJobDetailsPlugin.py +++ b/cuegui/cuegui/plugins/MonitorJobDetailsPlugin.py @@ -22,8 +22,8 @@ from __future__ import division from __future__ import absolute_import -from PySide2 import QtCore -from PySide2 import QtWidgets +from qtpy import QtCore +from qtpy import QtWidgets import opencue diff --git a/cuegui/cuegui/plugins/MonitorJobsPlugin.py b/cuegui/cuegui/plugins/MonitorJobsPlugin.py index f9ab677c2..a8ee4e034 100644 --- a/cuegui/cuegui/plugins/MonitorJobsPlugin.py +++ b/cuegui/cuegui/plugins/MonitorJobsPlugin.py @@ -26,9 +26,9 @@ import re import weakref -from PySide2 import QtGui -from PySide2 import QtCore -from PySide2 import QtWidgets +from qtpy import QtGui +from qtpy import QtCore +from qtpy import QtWidgets import opencue diff --git a/cuegui/cuegui/plugins/ShowsPlugin.py b/cuegui/cuegui/plugins/ShowsPlugin.py index deda343f9..9d292a5a8 100644 --- a/cuegui/cuegui/plugins/ShowsPlugin.py +++ b/cuegui/cuegui/plugins/ShowsPlugin.py @@ -20,7 +20,7 @@ from __future__ import division from __future__ import absolute_import -from PySide2 import QtWidgets +from qtpy import QtWidgets import cuegui.AbstractDockWidget import cuegui.CreateShowDialog diff --git a/cuegui/cuegui/plugins/StuckFramePlugin.py b/cuegui/cuegui/plugins/StuckFramePlugin.py index f3d5e2904..a44c90196 100644 --- a/cuegui/cuegui/plugins/StuckFramePlugin.py +++ b/cuegui/cuegui/plugins/StuckFramePlugin.py @@ -31,9 +31,9 @@ import signal import yaml -from PySide2 import QtGui -from PySide2 import QtCore -from PySide2 import QtWidgets +from qtpy import QtGui +from qtpy import QtCore +from qtpy import QtWidgets import opencue.wrappers.frame diff --git a/cuegui/cuegui/plugins/SubscriptionsGraphPlugin.py b/cuegui/cuegui/plugins/SubscriptionsGraphPlugin.py index 80dde33ca..a8c0078ef 100644 --- a/cuegui/cuegui/plugins/SubscriptionsGraphPlugin.py +++ b/cuegui/cuegui/plugins/SubscriptionsGraphPlugin.py @@ -20,8 +20,8 @@ from __future__ import division from __future__ import print_function -from PySide2 import QtCore -from PySide2 import QtWidgets +from qtpy import QtCore +from qtpy import QtWidgets import cuegui.AbstractDockWidget import cuegui.SubscriptionGraphWidget diff --git a/cuegui/cuegui/plugins/SubscriptionsPlugin.py b/cuegui/cuegui/plugins/SubscriptionsPlugin.py index 7c0d11731..e40081802 100644 --- a/cuegui/cuegui/plugins/SubscriptionsPlugin.py +++ b/cuegui/cuegui/plugins/SubscriptionsPlugin.py @@ -20,8 +20,8 @@ from __future__ import division from __future__ import absolute_import -from PySide2 import QtCore -from PySide2 import QtWidgets +from qtpy import QtCore +from qtpy import QtWidgets import cuegui.AbstractDockWidget import cuegui.SubscriptionsWidget diff --git a/cuegui/setup.py b/cuegui/setup.py index 79882eda3..2068936e0 100644 --- a/cuegui/setup.py +++ b/cuegui/setup.py @@ -65,6 +65,7 @@ 'grpcio-tools', 'PySide2', 'PyYAML', + 'QtPy', ] ) diff --git a/cuegui/tests/CueJobMonitorTree_tests.py b/cuegui/tests/CueJobMonitorTree_tests.py index 2cf93d0d5..2698ccedc 100644 --- a/cuegui/tests/CueJobMonitorTree_tests.py +++ b/cuegui/tests/CueJobMonitorTree_tests.py @@ -19,9 +19,9 @@ import unittest import mock -import PySide2.QtCore -import PySide2.QtGui -import PySide2.QtWidgets +import qtpy.QtCore +import qtpy.QtGui +import qtpy.QtWidgets import opencue.compiled_proto.job_pb2 import opencue.compiled_proto.show_pb2 @@ -40,7 +40,7 @@ class CueJobMonitorTreeTests(unittest.TestCase): @mock.patch('opencue.cuebot.Cuebot.getStub') def setUp(self, get_stub_mock): app = test_utils.createApplication() - app.settings = PySide2.QtCore.QSettings() + app.settings = qtpy.QtCore.QSettings() cuegui.Style.init() self.show_name = 'arbitrary-show-name' @@ -60,7 +60,7 @@ def setUp(self, get_stub_mock): name=self.show_name, jobs=self.jobs)) - self.main_window = PySide2.QtWidgets.QMainWindow() + self.main_window = qtpy.QtWidgets.QMainWindow() self.widget = cuegui.plugins.MonitorCuePlugin.MonitorCueDockWidget(self.main_window) self.cue_job_monitor_tree = cuegui.CueJobMonitorTree.CueJobMonitorTree(self.widget) self.cue_job_monitor_tree.addShow(self.show_name) diff --git a/cuegui/tests/DependWizard_tests.py b/cuegui/tests/DependWizard_tests.py index de1632502..5efb49f74 100644 --- a/cuegui/tests/DependWizard_tests.py +++ b/cuegui/tests/DependWizard_tests.py @@ -19,10 +19,10 @@ import unittest import mock -import PySide2.QtCore -import PySide2.QtGui -import PySide2.QtWidgets -import PySide2.QtTest +import qtpy.QtCore +import qtpy.QtGui +import qtpy.QtWidgets +import qtpy.QtTest import opencue.compiled_proto.job_pb2 import opencue.wrappers.frame @@ -41,10 +41,10 @@ class DependWizardTests(unittest.TestCase): @mock.patch('opencue.cuebot.Cuebot.getStub', new=mock.Mock()) def setUp(self): app = test_utils.createApplication() - app.settings = PySide2.QtCore.QSettings() + app.settings = qtpy.QtCore.QSettings() cuegui.Style.init() - self.parentWidget = PySide2.QtWidgets.QWidget() + self.parentWidget = qtpy.QtWidgets.QWidget() @mock.patch('cuegui.Cuedepend.createJobOnLayerDepend') @mock.patch('opencue.api.findJob') diff --git a/cuegui/tests/FilterDialog_tests.py b/cuegui/tests/FilterDialog_tests.py index bfe4c5832..536d65ec4 100644 --- a/cuegui/tests/FilterDialog_tests.py +++ b/cuegui/tests/FilterDialog_tests.py @@ -20,10 +20,10 @@ import mock -import PySide2.QtCore -import PySide2.QtGui -import PySide2.QtWidgets -import PySide2.QtTest +import qtpy.QtCore +import qtpy.QtGui +import qtpy.QtWidgets +import qtpy.QtTest import opencue.compiled_proto.show_pb2 import opencue.compiled_proto.filter_pb2 @@ -42,7 +42,7 @@ class FilterDialogTests(unittest.TestCase): @mock.patch('opencue.cuebot.Cuebot.getStub') def setUp(self, getStubMock): app = test_utils.createApplication() - app.settings = PySide2.QtCore.QSettings() + app.settings = qtpy.QtCore.QSettings() cuegui.Style.init() self.show = opencue.wrappers.show.Show(opencue.compiled_proto.show_pb2.Show(name='fooShow')) @@ -54,7 +54,7 @@ def setUp(self, getStubMock): opencue.compiled_proto.show_pb2.ShowGetFiltersResponse( filters=opencue.compiled_proto.filter_pb2.FilterSeq(filters=[filterProto])) - self.parentWidget = PySide2.QtWidgets.QWidget() + self.parentWidget = qtpy.QtWidgets.QWidget() self.filterDialog = cuegui.FilterDialog.FilterDialog(self.show, parent=self.parentWidget) def test_shouldTriggerRefresh(self): @@ -64,7 +64,7 @@ def test_shouldTriggerRefresh(self): self.show.getFilters.assert_called() - @mock.patch('PySide2.QtWidgets.QInputDialog.getText') + @mock.patch('qtpy.QtWidgets.QInputDialog.getText') def test_shouldAddFilter(self, getTextMock): newFilterId = 'new-filter-id' newFilterName = 'new-filter-name' @@ -78,7 +78,7 @@ def test_shouldAddFilter(self, getTextMock): self.show.createFilter.assert_called_with(newFilterName) - @mock.patch('PySide2.QtWidgets.QInputDialog.getText') + @mock.patch('qtpy.QtWidgets.QInputDialog.getText') def test_shouldCancelAddingFilter(self, getTextMock): self.show.createFilter = mock.Mock() getTextMock.return_value = (None, False) @@ -142,11 +142,11 @@ def test_shouldTriggerCreateAction(self): self.filterDialog._FilterDialog__actions.createAction.assert_called() def test_shouldCloseDialog(self): - self.assertEqual(PySide2.QtWidgets.QDialog.DialogCode.Rejected, self.filterDialog.result()) + self.assertEqual(qtpy.QtWidgets.QDialog.DialogCode.Rejected, self.filterDialog.result()) self.filterDialog._FilterDialog__btnDone.click() - self.assertEqual(PySide2.QtWidgets.QDialog.DialogCode.Accepted, self.filterDialog.result()) + self.assertEqual(qtpy.QtWidgets.QDialog.DialogCode.Accepted, self.filterDialog.result()) @mock.patch('opencue.cuebot.Cuebot.getStub', new=mock.Mock()) @@ -155,7 +155,7 @@ class FilterMonitorTreeTests(unittest.TestCase): @mock.patch('opencue.cuebot.Cuebot.getStub') def setUp(self, getStubMock): app = test_utils.createApplication() - app.settings = PySide2.QtCore.QSettings() + app.settings = qtpy.QtCore.QSettings() cuegui.Style.init() show = opencue.wrappers.show.Show(opencue.compiled_proto.show_pb2.Show(name='fooShow')) @@ -169,7 +169,7 @@ def setUp(self, getStubMock): opencue.compiled_proto.show_pb2.ShowGetFiltersResponse( filters=opencue.compiled_proto.filter_pb2.FilterSeq(filters=filters)) - self.parentWidget = PySide2.QtWidgets.QWidget() + self.parentWidget = qtpy.QtWidgets.QWidget() self.filterDialog = cuegui.FilterDialog.FilterDialog(show, parent=self.parentWidget) self.filterMonitorTree = self.filterDialog._FilterDialog__filters @@ -182,13 +182,13 @@ def test_shouldPopulateFiltersList(self): self.assertEqual('2', secondItem.text(0)) self.assertEqual(False, self.filterMonitorTree.itemWidget(secondItem, 1).isChecked()) - @mock.patch('PySide2.QtWidgets.QMenu') + @mock.patch('qtpy.QtWidgets.QMenu') def test_shouldRaiseContextMenu(self, qMenuMock): filterBeingSelected = self.filterMonitorTree.topLevelItem(0) self.filterMonitorTree.contextMenuEvent( - PySide2.QtGui.QContextMenuEvent( - PySide2.QtGui.QContextMenuEvent.Reason.Mouse, + qtpy.QtGui.QContextMenuEvent( + qtpy.QtGui.QContextMenuEvent.Reason.Mouse, self.filterMonitorTree.visualItemRect(filterBeingSelected).center())) qMenuMock.return_value.exec_.assert_called() @@ -200,7 +200,7 @@ class MatcherMonitorTreeTests(unittest.TestCase): @mock.patch('opencue.cuebot.Cuebot.getStub') def setUp(self, getStubMock): app = test_utils.createApplication() - app.settings = PySide2.QtCore.QSettings() + app.settings = qtpy.QtCore.QSettings() cuegui.Style.init() self.matchers = [ @@ -219,7 +219,7 @@ def setUp(self, getStubMock): opencue.wrappers.filter.Matcher(matcher) for matcher in self.matchers] self.filter = opencue.wrappers.filter.Filter(opencue.compiled_proto.filter_pb2.Filter()) - self.parentWidget = PySide2.QtWidgets.QWidget() + self.parentWidget = qtpy.QtWidgets.QWidget() self.matcherMonitorTree = cuegui.FilterDialog.MatcherMonitorTree(None, self.parentWidget) def test_shouldPopulateMatchersList(self): @@ -238,8 +238,8 @@ def test_shouldPopulateMatchersList(self): self.assertEqual('IS', self.matcherMonitorTree.itemWidget(secondItem, 1).currentText()) self.assertEqual('showName', self.matcherMonitorTree.itemWidget(secondItem, 2).text()) - @mock.patch('PySide2.QtWidgets.QInputDialog.getText') - @mock.patch('PySide2.QtWidgets.QInputDialog.getItem') + @mock.patch('qtpy.QtWidgets.QInputDialog.getText') + @mock.patch('qtpy.QtWidgets.QInputDialog.getItem') def test_shouldAddMatcher(self, getItemMock, getTextMock): matcherSubject = opencue.compiled_proto.filter_pb2.FACILITY matcherType = opencue.compiled_proto.filter_pb2.CONTAINS @@ -270,8 +270,8 @@ def test_shouldAddMatcher(self, getItemMock, getTextMock): 'CONTAINS', self.matcherMonitorTree.itemWidget(matcherWidget, 1).currentText()) self.assertEqual(matcherText, self.matcherMonitorTree.itemWidget(matcherWidget, 2).text()) - @mock.patch('PySide2.QtWidgets.QInputDialog.getText') - @mock.patch('PySide2.QtWidgets.QInputDialog.getItem') + @mock.patch('qtpy.QtWidgets.QInputDialog.getText') + @mock.patch('qtpy.QtWidgets.QInputDialog.getItem') def test_shouldCancelMatcherAdditionAtFirstPrompt(self, getItemMock, getTextMock): self.filter.createMatcher = mock.Mock() getItemMock.side_effect = [ @@ -285,8 +285,8 @@ def test_shouldCancelMatcherAdditionAtFirstPrompt(self, getItemMock, getTextMock self.filter.createMatcher.assert_not_called() - @mock.patch('PySide2.QtWidgets.QInputDialog.getText') - @mock.patch('PySide2.QtWidgets.QInputDialog.getItem') + @mock.patch('qtpy.QtWidgets.QInputDialog.getText') + @mock.patch('qtpy.QtWidgets.QInputDialog.getItem') def test_shouldCancelMatcherAdditionAtSecondPrompt(self, getItemMock, getTextMock): self.filter.createMatcher = mock.Mock() getItemMock.side_effect = [ @@ -300,8 +300,8 @@ def test_shouldCancelMatcherAdditionAtSecondPrompt(self, getItemMock, getTextMoc self.filter.createMatcher.assert_not_called() - @mock.patch('PySide2.QtWidgets.QInputDialog.getText') - @mock.patch('PySide2.QtWidgets.QInputDialog.getItem') + @mock.patch('qtpy.QtWidgets.QInputDialog.getText') + @mock.patch('qtpy.QtWidgets.QInputDialog.getItem') def test_shouldCancelMatcherAdditionAtThirdrompt(self, getItemMock, getTextMock): self.filter.createMatcher = mock.Mock() getItemMock.side_effect = [ @@ -316,8 +316,8 @@ def test_shouldCancelMatcherAdditionAtThirdrompt(self, getItemMock, getTextMock) self.filter.createMatcher.assert_not_called() @mock.patch( - 'PySide2.QtWidgets.QMessageBox.question', - new=mock.Mock(return_value=PySide2.QtWidgets.QMessageBox.Yes)) + 'qtpy.QtWidgets.QMessageBox.question', + new=mock.Mock(return_value=qtpy.QtWidgets.QMessageBox.Yes)) def test_shouldDeleteAllMatchers(self): self.filter.getMatchers = mock.Mock(return_value=self.matcherWrappers) for matcher in self.matcherWrappers: @@ -330,8 +330,8 @@ def test_shouldDeleteAllMatchers(self): matcher.delete.assert_called() @mock.patch( - 'PySide2.QtWidgets.QMessageBox.question', - new=mock.Mock(return_value=PySide2.QtWidgets.QMessageBox.No)) + 'qtpy.QtWidgets.QMessageBox.question', + new=mock.Mock(return_value=qtpy.QtWidgets.QMessageBox.No)) def test_shouldNotDeleteAnyMatchers(self): self.filter.getMatchers = mock.Mock(return_value=self.matcherWrappers) for matcher in self.matcherWrappers: @@ -345,7 +345,7 @@ def test_shouldNotDeleteAnyMatchers(self): @mock.patch('cuegui.Utils.questionBoxYesNo', new=mock.Mock(return_value=True)) @mock.patch('cuegui.TextEditDialog.TextEditDialog') - @mock.patch('PySide2.QtWidgets.QInputDialog.getItem') + @mock.patch('qtpy.QtWidgets.QInputDialog.getItem') def test_shouldAddMultipleMatchers(self, getItemMock, textEditDialogMock): matcherSubject = opencue.compiled_proto.filter_pb2.SHOT matcherType = opencue.compiled_proto.filter_pb2.IS @@ -382,7 +382,7 @@ def test_shouldAddMultipleMatchers(self, getItemMock, textEditDialogMock): @mock.patch('cuegui.Utils.questionBoxYesNo', new=mock.Mock(return_value=True)) @mock.patch('cuegui.TextEditDialog.TextEditDialog') - @mock.patch('PySide2.QtWidgets.QInputDialog.getItem') + @mock.patch('qtpy.QtWidgets.QInputDialog.getItem') def test_shouldReplaceAllMatchers(self, getItemMock, textEditDialogMock): matcherSubject = opencue.compiled_proto.filter_pb2.SHOT matcherType = opencue.compiled_proto.filter_pb2.IS diff --git a/cuegui/tests/FrameMonitorTree_tests.py b/cuegui/tests/FrameMonitorTree_tests.py index fc77e864c..75521572f 100644 --- a/cuegui/tests/FrameMonitorTree_tests.py +++ b/cuegui/tests/FrameMonitorTree_tests.py @@ -19,10 +19,10 @@ import unittest import mock -import PySide2.QtCore -import PySide2.QtGui -import PySide2.QtTest -import PySide2.QtWidgets +import qtpy.QtCore +import qtpy.QtGui +import qtpy.QtTest +import qtpy.QtWidgets import opencue.compiled_proto.job_pb2 import opencue.wrappers.frame @@ -44,9 +44,9 @@ class FrameMonitorTreeTests(unittest.TestCase): @mock.patch('opencue.cuebot.Cuebot.getStub', new=mock.Mock()) def setUp(self): app = test_utils.createApplication() - app.settings = PySide2.QtCore.QSettings() + app.settings = qtpy.QtCore.QSettings() cuegui.Style.init() - self.parentWidget = PySide2.QtWidgets.QWidget() + self.parentWidget = qtpy.QtWidgets.QWidget() self.frameMonitorTree = cuegui.FrameMonitorTree.FrameMonitorTree(self.parentWidget) self.job = opencue.wrappers.job.Job(opencue.compiled_proto.job_pb2.Job(id='foo')) self.frameMonitorTree.setJob(self.job) @@ -126,11 +126,11 @@ def test_getCores(self): @mock.patch.object(cuegui.FrameMonitorTree.FrameContextMenu, 'exec_') def test_rightClickItem(self, execMock): - mouse_position = PySide2.QtCore.QPoint() + mouse_position = qtpy.QtCore.QPoint() self.frameMonitorTree.contextMenuEvent( - PySide2.QtGui.QContextMenuEvent( - PySide2.QtGui.QContextMenuEvent.Reason.Mouse, mouse_position, mouse_position)) + qtpy.QtGui.QContextMenuEvent( + qtpy.QtGui.QContextMenuEvent.Reason.Mouse, mouse_position, mouse_position)) execMock.assert_called_with(mouse_position) @@ -152,7 +152,7 @@ def setUp(self): checkpoint_state=opencue.compiled_proto.job_pb2.ENABLED)) # The widget needs a var, otherwise it gets garbage-collected before tests can run. - parentWidget = PySide2.QtWidgets.QWidget() + parentWidget = qtpy.QtWidgets.QWidget() self.frameWidgetItem = cuegui.FrameMonitorTree.FrameWidgetItem( self.frame, @@ -165,46 +165,46 @@ def test_data(self): self.assertEqual( self.dispatch_order, - self.frameWidgetItem.data(dispatch_order_col, PySide2.QtCore.Qt.DisplayRole)) + self.frameWidgetItem.data(dispatch_order_col, qtpy.QtCore.Qt.DisplayRole)) self.assertEqual( cuegui.Style.ColorTheme.COLOR_JOB_FOREGROUND, - self.frameWidgetItem.data(dispatch_order_col, PySide2.QtCore.Qt.ForegroundRole)) + self.frameWidgetItem.data(dispatch_order_col, qtpy.QtCore.Qt.ForegroundRole)) self.assertEqual( cuegui.FrameMonitorTree.QCOLOR_BLACK, self.frameWidgetItem.data( - cuegui.FrameMonitorTree.STATUS_COLUMN, PySide2.QtCore.Qt.ForegroundRole)) + cuegui.FrameMonitorTree.STATUS_COLUMN, qtpy.QtCore.Qt.ForegroundRole)) self.assertEqual( cuegui.FrameMonitorTree.QCOLOR_GREEN, self.frameWidgetItem.data( - cuegui.FrameMonitorTree.PROC_COLUMN, PySide2.QtCore.Qt.ForegroundRole)) + cuegui.FrameMonitorTree.PROC_COLUMN, qtpy.QtCore.Qt.ForegroundRole)) self.assertEqual( cuegui.Constants.RGB_FRAME_STATE[self.state], self.frameWidgetItem.data( - cuegui.FrameMonitorTree.STATUS_COLUMN, PySide2.QtCore.Qt.BackgroundRole)) + cuegui.FrameMonitorTree.STATUS_COLUMN, qtpy.QtCore.Qt.BackgroundRole)) self.assertEqual( - PySide2.QtGui.QIcon, + qtpy.QtGui.QIcon, self.frameWidgetItem.data( cuegui.FrameMonitorTree.CHECKPOINT_COLUMN, - PySide2.QtCore.Qt.DecorationRole).__class__) + qtpy.QtCore.Qt.DecorationRole).__class__) self.assertEqual( - PySide2.QtCore.Qt.AlignCenter, + qtpy.QtCore.Qt.AlignCenter, self.frameWidgetItem.data( - cuegui.FrameMonitorTree.STATUS_COLUMN, PySide2.QtCore.Qt.TextAlignmentRole)) + cuegui.FrameMonitorTree.STATUS_COLUMN, qtpy.QtCore.Qt.TextAlignmentRole)) self.assertEqual( - PySide2.QtCore.Qt.AlignRight, + qtpy.QtCore.Qt.AlignRight, self.frameWidgetItem.data( - cuegui.FrameMonitorTree.PROC_COLUMN, PySide2.QtCore.Qt.TextAlignmentRole)) + cuegui.FrameMonitorTree.PROC_COLUMN, qtpy.QtCore.Qt.TextAlignmentRole)) self.assertEqual( cuegui.Constants.TYPE_FRAME, - self.frameWidgetItem.data(dispatch_order_col, PySide2.QtCore.Qt.UserRole)) + self.frameWidgetItem.data(dispatch_order_col, qtpy.QtCore.Qt.UserRole)) if __name__ == '__main__': diff --git a/cuegui/tests/LayerDialog_tests.py b/cuegui/tests/LayerDialog_tests.py index 78d4d5060..f50e9cfab 100644 --- a/cuegui/tests/LayerDialog_tests.py +++ b/cuegui/tests/LayerDialog_tests.py @@ -20,9 +20,9 @@ import mock -import PySide2.QtCore -import PySide2.QtGui -import PySide2.QtWidgets +import qtpy.QtCore +import qtpy.QtGui +import qtpy.QtWidgets import opencue.compiled_proto.show_pb2 import opencue.compiled_proto.filter_pb2 @@ -48,7 +48,7 @@ class LayerPropertiesDialogTests(unittest.TestCase): @mock.patch('opencue.cuebot.Cuebot.getStub') def setUp(self, get_stub_mock, get_layer_mock, get_limits_mock): app = test_utils.createApplication() - app.settings = PySide2.QtCore.QSettings() + app.settings = qtpy.QtCore.QSettings() cuegui.Style.init() self.layers = { @@ -80,7 +80,7 @@ def setUp(self, get_stub_mock, get_layer_mock, get_limits_mock): opencue.compiled_proto.limit_pb2.Limit(id='limit4Id', name='limit4Name')), ] - self.parent_widget = PySide2.QtWidgets.QWidget() + self.parent_widget = qtpy.QtWidgets.QWidget() self.layer_properties_dialog = cuegui.LayerDialog.LayerPropertiesDialog( ['layer1Id', 'layer2Id'], parent=self.parent_widget) diff --git a/cuegui/tests/Layout_tests.py b/cuegui/tests/Layout_tests.py index c291c3b1e..2b6818f9c 100644 --- a/cuegui/tests/Layout_tests.py +++ b/cuegui/tests/Layout_tests.py @@ -25,7 +25,7 @@ import tempfile import unittest -from PySide2 import QtCore +from qtpy import QtCore import cuegui.Layout diff --git a/cuegui/tests/MenuActions_tests.py b/cuegui/tests/MenuActions_tests.py index 0127cc7ce..e65c590da 100644 --- a/cuegui/tests/MenuActions_tests.py +++ b/cuegui/tests/MenuActions_tests.py @@ -23,8 +23,8 @@ import unittest import mock -import PySide2.QtGui -import PySide2.QtWidgets +import qtpy.QtGui +import qtpy.QtWidgets import opencue.compiled_proto.depend_pb2 import opencue.compiled_proto.facility_pb2 @@ -101,7 +101,7 @@ def test_emailArtist(self, emailDialogMock): emailDialogMock.assert_called_with(job, self.widgetMock) emailDialogMock.return_value.show.assert_called() - @mock.patch('PySide2.QtWidgets.QInputDialog.getDouble') + @mock.patch('qtpy.QtWidgets.QInputDialog.getDouble') def test_setMinCores(self, getDoubleMock): highest_current_core_count = 20 new_core_count = 50 @@ -123,7 +123,7 @@ def test_setMinCores(self, getDoubleMock): job1.setMinCores.assert_called_with(new_core_count) job2.setMinCores.assert_called_with(new_core_count) - @mock.patch('PySide2.QtWidgets.QInputDialog.getDouble') + @mock.patch('qtpy.QtWidgets.QInputDialog.getDouble') def test_setMinCoresCanceled(self, getDoubleMock): job1 = opencue.wrappers.job.Job(opencue.compiled_proto.job_pb2.Job(min_cores=0)) job1.setMinCores = mock.Mock() @@ -136,7 +136,7 @@ def test_setMinCoresCanceled(self, getDoubleMock): job1.setMinCores.assert_not_called() job2.setMinCores.assert_not_called() - @mock.patch('PySide2.QtWidgets.QInputDialog.getDouble') + @mock.patch('qtpy.QtWidgets.QInputDialog.getDouble') def test_setMaxCores(self, getDoubleMock): highest_current_core_count = 20 new_core_count = 50 @@ -158,7 +158,7 @@ def test_setMaxCores(self, getDoubleMock): job1.setMaxCores.assert_called_with(new_core_count) job2.setMaxCores.assert_called_with(new_core_count) - @mock.patch('PySide2.QtWidgets.QInputDialog.getDouble') + @mock.patch('qtpy.QtWidgets.QInputDialog.getDouble') def test_setMaxCoresCanceled(self, getDoubleMock): job1 = opencue.wrappers.job.Job(opencue.compiled_proto.job_pb2.Job(max_cores=0)) job1.setMaxCores = mock.Mock() @@ -171,7 +171,7 @@ def test_setMaxCoresCanceled(self, getDoubleMock): job1.setMaxCores.assert_not_called() job2.setMaxCores.assert_not_called() - @mock.patch('PySide2.QtWidgets.QInputDialog.getInt') + @mock.patch('qtpy.QtWidgets.QInputDialog.getInt') def test_setPriority(self, getIntMock): job = opencue.wrappers.job.Job(opencue.compiled_proto.job_pb2.Job(priority=0)) job.setPriority = mock.Mock() @@ -182,7 +182,7 @@ def test_setPriority(self, getIntMock): job.setPriority.assert_called_with(new_priority) - @mock.patch('PySide2.QtWidgets.QInputDialog.getInt') + @mock.patch('qtpy.QtWidgets.QInputDialog.getInt') def test_setPriorityCanceled(self, getIntMock): job = opencue.wrappers.job.Job(opencue.compiled_proto.job_pb2.Job(priority=0)) job.setPriority = mock.Mock() @@ -192,7 +192,7 @@ def test_setPriorityCanceled(self, getIntMock): job.setPriority.assert_not_called() - @mock.patch('PySide2.QtWidgets.QInputDialog.getInt') + @mock.patch('qtpy.QtWidgets.QInputDialog.getInt') def test_setMaxRetries(self, getIntMock): job = opencue.wrappers.job.Job() job.setMaxRetries = mock.Mock() @@ -203,7 +203,7 @@ def test_setMaxRetries(self, getIntMock): job.setMaxRetries.assert_called_with(new_retries) - @mock.patch('PySide2.QtWidgets.QInputDialog.getInt') + @mock.patch('qtpy.QtWidgets.QInputDialog.getInt') def test_setMaxRetriesCanceled(self, getIntMock): job = opencue.wrappers.job.Job() job.setMaxRetries = mock.Mock() @@ -356,8 +356,8 @@ def test_dependWizard(self, dependWizardMock): dependWizardMock.assert_called_with(self.widgetMock, jobs) - @mock.patch('PySide2.QtWidgets.QInputDialog.getItem') - @mock.patch('PySide2.QtWidgets.QInputDialog.getText') + @mock.patch('qtpy.QtWidgets.QInputDialog.getItem') + @mock.patch('qtpy.QtWidgets.QInputDialog.getText') def test_reorder(self, getTextMock, getItemMock): original_range = '1-10' new_order = 'REVERSE' @@ -373,8 +373,8 @@ def test_reorder(self, getTextMock, getItemMock): job.reorderFrames.assert_called_with(original_range, opencue.compiled_proto.job_pb2.REVERSE) - @mock.patch('PySide2.QtWidgets.QInputDialog.getItem') - @mock.patch('PySide2.QtWidgets.QInputDialog.getText') + @mock.patch('qtpy.QtWidgets.QInputDialog.getItem') + @mock.patch('qtpy.QtWidgets.QInputDialog.getText') def test_reorderCanceled(self, getTextMock, getItemMock): original_range = '1-10' job = opencue.wrappers.job.Job(opencue.compiled_proto.job_pb2.Job(name='job-name')) @@ -397,8 +397,8 @@ def test_reorderCanceled(self, getTextMock, getItemMock): job.reorderFrames.assert_not_called() - @mock.patch('PySide2.QtWidgets.QInputDialog.getInt') - @mock.patch('PySide2.QtWidgets.QInputDialog.getText') + @mock.patch('qtpy.QtWidgets.QInputDialog.getInt') + @mock.patch('qtpy.QtWidgets.QInputDialog.getText') def test_stagger(self, getTextMock, getIntMock): original_range = '1-10' new_step = 28 @@ -414,8 +414,8 @@ def test_stagger(self, getTextMock, getIntMock): job.staggerFrames.assert_called_with(original_range, new_step) - @mock.patch('PySide2.QtWidgets.QInputDialog.getInt') - @mock.patch('PySide2.QtWidgets.QInputDialog.getText') + @mock.patch('qtpy.QtWidgets.QInputDialog.getInt') + @mock.patch('qtpy.QtWidgets.QInputDialog.getText') def test_staggerCanceled(self, getTextMock, getIntMock): original_range = '1-10' job = opencue.wrappers.job.Job(opencue.compiled_proto.job_pb2.Job(name='job-name')) @@ -507,7 +507,7 @@ def test_useLocalCores(self, localBookingDialogMock): localBookingDialogMock.assert_called_with(job, self.widgetMock) localBookingDialogMock.return_value.exec_.assert_called() - @mock.patch('PySide2.QtWidgets.QApplication.clipboard') + @mock.patch('qtpy.QtWidgets.QApplication.clipboard') def test_copyLogFileDir(self, clipboardMock): logDir1 = '/some/random/dir' logDir2 = '/a/different/random/dir' @@ -571,7 +571,7 @@ def test_viewDepends(self, dependDialogMock): dependDialogMock.return_value.show.assert_called() @mock.patch.object(opencue.wrappers.layer.Layer, 'setMinCores', autospec=True) - @mock.patch('PySide2.QtWidgets.QInputDialog.getDouble') + @mock.patch('qtpy.QtWidgets.QInputDialog.getDouble') def test_setMinCores(self, getDoubleMock, setMinCoresMock): highest_current_core_count = 20 new_core_count = 50 @@ -592,7 +592,7 @@ def test_setMinCores(self, getDoubleMock, setMinCoresMock): mock.call(layer1, new_core_count), mock.call(layer2, new_core_count)]) @mock.patch.object(opencue.wrappers.layer.Layer, 'setMinCores') - @mock.patch('PySide2.QtWidgets.QInputDialog.getDouble') + @mock.patch('qtpy.QtWidgets.QInputDialog.getDouble') def test_setMinCoresCanceled(self, getDoubleMock, setMinCoresMock): layer1 = opencue.wrappers.layer.Layer( opencue.compiled_proto.job_pb2.Layer(min_cores=0)) @@ -605,7 +605,7 @@ def test_setMinCoresCanceled(self, getDoubleMock, setMinCoresMock): setMinCoresMock.assert_not_called() @mock.patch.object(opencue.wrappers.layer.Layer, 'setMinMemory', autospec=True) - @mock.patch('PySide2.QtWidgets.QInputDialog.getDouble') + @mock.patch('qtpy.QtWidgets.QInputDialog.getDouble') def test_setMinMemoryKb(self, getDoubleMock, setMinMemoryMock): highest_current_mem_limit_gb = 20 new_mem_limit_gb = 50 @@ -630,7 +630,7 @@ def test_setMinMemoryKb(self, getDoubleMock, setMinMemoryMock): ]) @mock.patch.object(opencue.wrappers.layer.Layer, 'setMinMemory') - @mock.patch('PySide2.QtWidgets.QInputDialog.getDouble') + @mock.patch('qtpy.QtWidgets.QInputDialog.getDouble') def test_setMinMemoryKbCanceled(self, getDoubleMock, setMinMemoryMock): layer1 = opencue.wrappers.layer.Layer(opencue.compiled_proto.job_pb2.Layer(min_memory=0)) layer2 = opencue.wrappers.layer.Layer(opencue.compiled_proto.job_pb2.Layer(min_memory=0)) @@ -758,8 +758,8 @@ def test_dependWizard(self, dependWizardMock): dependWizardMock.assert_called_with(self.widgetMock, [self.job], layers=layers) @mock.patch.object(opencue.wrappers.layer.Layer, 'reorderFrames', autospec=True) - @mock.patch('PySide2.QtWidgets.QInputDialog.getItem') - @mock.patch('PySide2.QtWidgets.QInputDialog.getText') + @mock.patch('qtpy.QtWidgets.QInputDialog.getItem') + @mock.patch('qtpy.QtWidgets.QInputDialog.getText') def test_reorder(self, getTextMock, getItemMock, reorderFramesMock): original_range = '1-10' new_order = 'REVERSE' @@ -775,8 +775,8 @@ def test_reorder(self, getTextMock, getItemMock, reorderFramesMock): layer, original_range, opencue.compiled_proto.job_pb2.REVERSE) @mock.patch.object(opencue.wrappers.layer.Layer, 'staggerFrames', autospec=True) - @mock.patch('PySide2.QtWidgets.QInputDialog.getInt') - @mock.patch('PySide2.QtWidgets.QInputDialog.getText') + @mock.patch('qtpy.QtWidgets.QInputDialog.getInt') + @mock.patch('qtpy.QtWidgets.QInputDialog.getText') def test_stagger(self, getTextMock, getIntMock, staggerFramesMock): original_range = '1-10' new_step = 28 @@ -998,7 +998,7 @@ def test_markdone(self, yesNoMock): self.job.markdoneFrames.assert_called_with(name=[frame_name]) @mock.patch.object(opencue.wrappers.layer.Layer, 'reorderFrames', autospec=True) - @mock.patch('PySide2.QtWidgets.QInputDialog.getItem') + @mock.patch('qtpy.QtWidgets.QInputDialog.getItem') def test_reorder(self, getItemMock, reorderFramesMock): new_order = 'REVERSE' getItemMock.return_value = (new_order, True) @@ -1015,7 +1015,7 @@ def test_reorder(self, getItemMock, reorderFramesMock): reorderFramesMock.assert_called_with( layer, str(frame_num), opencue.compiled_proto.job_pb2.REVERSE) - @mock.patch('PySide2.QtWidgets.QApplication.clipboard') + @mock.patch('qtpy.QtWidgets.QApplication.clipboard') @mock.patch('cuegui.Utils.getFrameLogFile') def test_copyLogFileName(self, getFrameLogFileMock, clipboardMock): frame_log_path = '/some/path/to/job/logs/job-name.frame-name.rqlog' @@ -1133,21 +1133,21 @@ def setUp(self): self.subscription_actions = cuegui.MenuActions.SubscriptionActions( self.widgetMock, mock.Mock(), None, None) - @mock.patch('PySide2.QtWidgets.QMessageBox') - @mock.patch('PySide2.QtWidgets.QInputDialog.getDouble') + @mock.patch('qtpy.QtWidgets.QMessageBox') + @mock.patch('qtpy.QtWidgets.QInputDialog.getDouble') def test_editSize(self, getDoubleMock, qMessageBoxMock): sub = opencue.wrappers.subscription.Subscription( opencue.compiled_proto.subscription_pb2.Subscription(size=382)) sub.setSize = mock.MagicMock() newSize = 8479 getDoubleMock.return_value = (newSize, True) - qMessageBoxMock.return_value.exec_.return_value = PySide2.QtWidgets.QMessageBox.Yes + qMessageBoxMock.return_value.exec_.return_value = qtpy.QtWidgets.QMessageBox.Yes self.subscription_actions.editSize(rpcObjects=[sub]) sub.setSize.assert_called_with(newSize*100.0) - @mock.patch('PySide2.QtWidgets.QInputDialog.getDouble') + @mock.patch('qtpy.QtWidgets.QInputDialog.getDouble') def test_editBurst(self, getDoubleMock): sub = opencue.wrappers.subscription.Subscription( opencue.compiled_proto.subscription_pb2.Subscription(burst=922)) @@ -1249,7 +1249,7 @@ def test_rebootWhenIdle(self): host.rebootWhenIdle.assert_called() - @mock.patch('PySide2.QtWidgets.QInputDialog.getText') + @mock.patch('qtpy.QtWidgets.QInputDialog.getText') def test_addTags(self, getTextMock): host = opencue.wrappers.host.Host( opencue.compiled_proto.host_pb2.Host(id='arbitrary-id')) @@ -1261,7 +1261,7 @@ def test_addTags(self, getTextMock): host.addTags.assert_called_with(['firstTag', 'anotherTag', 'oneMoreTag']) - @mock.patch('PySide2.QtWidgets.QInputDialog.getText') + @mock.patch('qtpy.QtWidgets.QInputDialog.getText') def test_removeTags(self, getTextMock): host = opencue.wrappers.host.Host( opencue.compiled_proto.host_pb2.Host( @@ -1273,8 +1273,8 @@ def test_removeTags(self, getTextMock): host.removeTags.assert_called_with(['firstTag', 'anotherTag', 'oneMoreTag']) - @mock.patch('PySide2.QtWidgets.QInputDialog.getText') - @mock.patch('PySide2.QtWidgets.QInputDialog.getItem') + @mock.patch('qtpy.QtWidgets.QInputDialog.getText') + @mock.patch('qtpy.QtWidgets.QInputDialog.getItem') def test_renameTag(self, getItemMock, getTextMock): host = opencue.wrappers.host.Host( opencue.compiled_proto.host_pb2.Host(id='arbitrary-id')) @@ -1288,7 +1288,7 @@ def test_renameTag(self, getItemMock, getTextMock): host.renameTag.assert_called_with(oldTagName, newTagName) - @mock.patch('PySide2.QtWidgets.QInputDialog.getItem') + @mock.patch('qtpy.QtWidgets.QInputDialog.getItem') @mock.patch('opencue.api.getAllocations') def test_changeAllocation(self, getAllocationsMock, getItemMock): host = opencue.wrappers.host.Host( @@ -1427,7 +1427,7 @@ def setUp(self): self.filter_actions = cuegui.MenuActions.FilterActions( self.widgetMock, mock.Mock(), None, None) - @mock.patch('PySide2.QtWidgets.QInputDialog.getText') + @mock.patch('qtpy.QtWidgets.QInputDialog.getText') def test_rename(self, getTextMock): filter_wrapper = opencue.wrappers.filter.Filter(opencue.compiled_proto.filter_pb2.Filter()) filter_wrapper.setName = mock.MagicMock() @@ -1479,7 +1479,7 @@ def test_orderLast(self): filter_wrapper.orderLast.assert_called() - @mock.patch('PySide2.QtWidgets.QInputDialog.getInt') + @mock.patch('qtpy.QtWidgets.QInputDialog.getInt') def test_setOrder(self, getTextMock): filter_wrapper = opencue.wrappers.filter.Filter(opencue.compiled_proto.filter_pb2.Filter()) filter_wrapper.setOrder = mock.MagicMock() @@ -1510,7 +1510,7 @@ def test_delete(self): matcher.delete.assert_called() - @mock.patch('PySide2.QtWidgets.QInputDialog.getText') + @mock.patch('qtpy.QtWidgets.QInputDialog.getText') def test_setValue(self, getTextMock): matcher = opencue.wrappers.filter.Matcher(opencue.compiled_proto.filter_pb2.Matcher()) matcher.setValue = mock.MagicMock() @@ -1552,7 +1552,7 @@ def setUp(self): self.task_actions = cuegui.MenuActions.TaskActions( self.widgetMock, mock.Mock(), None, None) - @mock.patch('PySide2.QtWidgets.QInputDialog.getDouble') + @mock.patch('qtpy.QtWidgets.QInputDialog.getDouble') def test_setMinCores(self, getDoubleMock): task = opencue.wrappers.task.Task(opencue.compiled_proto.task_pb2.Task(min_cores=10)) task.setMinCores = mock.MagicMock() @@ -1592,7 +1592,7 @@ def setUp(self): self.widgetMock, mock.Mock(), None, None) @mock.patch('opencue.api.createLimit') - @mock.patch('PySide2.QtWidgets.QInputDialog.getText') + @mock.patch('qtpy.QtWidgets.QInputDialog.getText') def test_create(self, getTextMock, createLimitMock): limitName = 'newLimitName' getTextMock.return_value = ('%s \t ' % limitName, True) @@ -1610,7 +1610,7 @@ def test_delete(self): limit.delete.assert_called() - @mock.patch('PySide2.QtWidgets.QInputDialog.getDouble') + @mock.patch('qtpy.QtWidgets.QInputDialog.getDouble') def test_editMaxValue(self, getDoubleMock): limit = opencue.wrappers.limit.Limit(opencue.compiled_proto.limit_pb2.Limit(max_value=920)) limit.setMaxValue = mock.MagicMock() @@ -1622,7 +1622,7 @@ def test_editMaxValue(self, getDoubleMock): limit.setMaxValue.assert_called_with(newMaxValue) - @mock.patch('PySide2.QtWidgets.QInputDialog.getText') + @mock.patch('qtpy.QtWidgets.QInputDialog.getText') def test_rename(self, getTextMock): limit = opencue.wrappers.limit.Limit(opencue.compiled_proto.limit_pb2.Limit()) limit.rename = mock.MagicMock() diff --git a/cuegui/tests/Redirect_tests.py b/cuegui/tests/Redirect_tests.py index 5e1f6ad27..ecfffcc48 100644 --- a/cuegui/tests/Redirect_tests.py +++ b/cuegui/tests/Redirect_tests.py @@ -19,8 +19,8 @@ import unittest import mock -import PySide2.QtCore -import PySide2.QtGui +import qtpy.QtCore +import qtpy.QtGui import opencue.compiled_proto.show_pb2 import opencue.wrappers.show @@ -37,7 +37,7 @@ class RedirectTests(unittest.TestCase): @mock.patch('opencue.cuebot.Cuebot.getStub') def setUp(self, getStubMock): app = test_utils.createApplication() - app.settings = PySide2.QtCore.QSettings() + app.settings = qtpy.QtCore.QSettings() cuegui.Style.init() getStubMock.return_value.GetActiveShows.return_value = \ diff --git a/cuegui/tests/UnbookDialog_tests.py b/cuegui/tests/UnbookDialog_tests.py index a62b390e7..bc5c348d8 100644 --- a/cuegui/tests/UnbookDialog_tests.py +++ b/cuegui/tests/UnbookDialog_tests.py @@ -20,8 +20,8 @@ import mock -import PySide2.QtCore -import PySide2.QtGui +import qtpy.QtCore +import qtpy.QtGui import opencue.compiled_proto.criterion_pb2 import opencue.compiled_proto.host_pb2 @@ -47,7 +47,7 @@ class UnbookDialogTests(unittest.TestCase): @mock.patch('opencue.cuebot.Cuebot.getStub') def setUp(self, get_stub_mock, find_show_mock): app = test_utils.createApplication() - app.settings = PySide2.QtCore.QSettings() + app.settings = qtpy.QtCore.QSettings() cuegui.Style.init() show_name = 'showname' @@ -85,7 +85,7 @@ def test__should_show_all_jobs_and_subscriptions(self): self.assertEqual(self.tag_names, subscriptions_shown) self.assertEqual(self.tag_names, subscriptions_checked) - @mock.patch('PySide2.QtWidgets.QMessageBox', new=mock.Mock()) + @mock.patch('qtpy.QtWidgets.QMessageBox', new=mock.Mock()) @mock.patch('opencue.api.getProcs') def test__should_unbook_procs(self, get_procs_mock): num_procs = 17 @@ -135,9 +135,9 @@ def test__should_show_kill_confirmation_dialog(self, kill_dialog_mock): kill_dialog_mock.assert_called_with(expected_proc_search, mock.ANY) - @mock.patch('PySide2.QtWidgets.QMessageBox', new=mock.Mock()) + @mock.patch('qtpy.QtWidgets.QMessageBox', new=mock.Mock()) @mock.patch('opencue.api.getProcs') - @mock.patch('PySide2.QtWidgets.QInputDialog.getItem') + @mock.patch('qtpy.QtWidgets.QInputDialog.getItem') @mock.patch('opencue.api.getActiveShows') def test__should_redirect_proc_to_group( self, get_active_shows_mock, get_item_mock, get_procs_mock): @@ -165,11 +165,11 @@ def test__should_redirect_proc_to_group( get_procs_mock.assert_called_with(**expected_proc_search.options) proc_to_redirect.redirectToGroup.assert_called_with(group, False) - @mock.patch('PySide2.QtWidgets.QMessageBox', new=mock.Mock()) + @mock.patch('qtpy.QtWidgets.QMessageBox', new=mock.Mock()) @mock.patch('opencue.api.getProcs') @mock.patch('cuegui.UnbookDialog.SelectItemsWithSearchDialog') @mock.patch('opencue.api.getJobs') - @mock.patch('PySide2.QtWidgets.QInputDialog.getItem') + @mock.patch('qtpy.QtWidgets.QInputDialog.getItem') @mock.patch('opencue.api.getActiveShows') def test__should_redirect_proc_to_job( self, get_active_shows_mock, get_item_mock, get_jobs_mock, select_job_mock, @@ -203,7 +203,7 @@ class SelectItemsWithSearchDialogTests(unittest.TestCase): def setUp(self): app = test_utils.createApplication() - app.settings = PySide2.QtCore.QSettings() + app.settings = qtpy.QtCore.QSettings() cuegui.Style.init() def test__should_display_all_items(self): @@ -246,10 +246,10 @@ class KillConfirmationDialogTests(unittest.TestCase): def setUp(self): app = test_utils.createApplication() - app.settings = PySide2.QtCore.QSettings() + app.settings = qtpy.QtCore.QSettings() cuegui.Style.init() - @mock.patch('PySide2.QtWidgets.QMessageBox.information', new=mock.Mock()) + @mock.patch('qtpy.QtWidgets.QMessageBox.information', new=mock.Mock()) @mock.patch('opencue.api.getProcs') def test__should_kill_procs(self, get_procs_mock): proc_search = opencue.search.ProcSearch( @@ -268,7 +268,7 @@ def test__should_kill_procs(self, get_procs_mock): proc1.kill.assert_called() proc2.kill.assert_called() - @mock.patch('PySide2.QtWidgets.QMessageBox.information', new=mock.Mock()) + @mock.patch('qtpy.QtWidgets.QMessageBox.information', new=mock.Mock()) @mock.patch('opencue.api.getProcs') def test__should_cancel_kill(self, get_procs_mock): proc_search = opencue.search.ProcSearch( diff --git a/cuegui/tests/plugins/LogViewPlugin_tests.py b/cuegui/tests/plugins/LogViewPlugin_tests.py index 8206176f8..a135747b5 100644 --- a/cuegui/tests/plugins/LogViewPlugin_tests.py +++ b/cuegui/tests/plugins/LogViewPlugin_tests.py @@ -22,10 +22,10 @@ import mock import pyfakefs.fake_filesystem_unittest -import PySide2.QtCore -import PySide2.QtGui -import PySide2.QtTest -import PySide2.QtWidgets +import qtpy.QtCore +import qtpy.QtGui +import qtpy.QtTest +import qtpy.QtWidgets import cuegui.Main import cuegui.plugins.LogViewPlugin @@ -64,9 +64,9 @@ def setUp(self): self.fs.create_file(self.logPath2, contents=_LOG_TEXT_2) test_utils.createApplication() - cuegui.app().settings = PySide2.QtCore.QSettings() + cuegui.app().settings = qtpy.QtCore.QSettings() cuegui.Style.init() - self.parentWidget = PySide2.QtWidgets.QMainWindow() + self.parentWidget = qtpy.QtWidgets.QMainWindow() self.logViewPlugin = cuegui.plugins.LogViewPlugin.LogViewPlugin(self.parentWidget) def test_shouldDisplayFirstLogFile(self): @@ -85,7 +85,7 @@ def test_shouldUpdateLogFile(self): def test_shouldHighlightAllSearchResults(self): cuegui.app().display_log_file_content.emit([self.logPath1, self.logPath2]) self.logViewPlugin.logview_widget._case_stv_checkbox.setCheckState( - PySide2.QtCore.Qt.CheckState.Unchecked) + qtpy.QtCore.Qt.CheckState.Unchecked) self.logViewPlugin.logview_widget._search_box.setText('lorem') self.logViewPlugin.logview_widget._search_button.click() @@ -102,7 +102,7 @@ def test_shouldHighlightAllSearchResults(self): def test_shouldMoveCursorToSecondSearchResult(self): cuegui.app().display_log_file_content.emit([self.logPath1, self.logPath2]) self.logViewPlugin.logview_widget._case_stv_checkbox.setCheckState( - PySide2.QtCore.Qt.CheckState.Unchecked) + qtpy.QtCore.Qt.CheckState.Unchecked) self.logViewPlugin.logview_widget._search_box.setText('lorem') self.logViewPlugin.logview_widget._search_button.click() @@ -116,7 +116,7 @@ def test_shouldMoveCursorToSecondSearchResult(self): def test_shouldMoveCursorLastSearchResult(self): cuegui.app().display_log_file_content.emit([self.logPath1, self.logPath2]) self.logViewPlugin.logview_widget._case_stv_checkbox.setCheckState( - PySide2.QtCore.Qt.CheckState.Unchecked) + qtpy.QtCore.Qt.CheckState.Unchecked) self.logViewPlugin.logview_widget._search_box.setText('lorem') self.logViewPlugin.logview_widget._search_button.click() @@ -130,7 +130,7 @@ def test_shouldMoveCursorLastSearchResult(self): def test_shouldPerformCaseInsensitiveSearch(self): cuegui.app().display_log_file_content.emit([self.logPath1, self.logPath2]) self.logViewPlugin.logview_widget._case_stv_checkbox.setCheckState( - PySide2.QtCore.Qt.CheckState.Checked) + qtpy.QtCore.Qt.CheckState.Checked) self.logViewPlugin.logview_widget._search_box.setText('lorem') self.logViewPlugin.logview_widget._search_button.click() @@ -143,12 +143,12 @@ def test_shouldPerformCaseInsensitiveSearch(self): @staticmethod def __isHighlighted(textBox, startPosition, selectionLength): - cursor = textBox.cursorForPosition(PySide2.QtCore.QPoint(0, 0)) + cursor = textBox.cursorForPosition(qtpy.QtCore.QPoint(0, 0)) cursor.setPosition(startPosition) - cursor.movePosition(PySide2.QtGui.QTextCursor.Right, - PySide2.QtGui.QTextCursor.KeepAnchor, + cursor.movePosition(qtpy.QtGui.QTextCursor.Right, + qtpy.QtGui.QTextCursor.KeepAnchor, selectionLength) - return cursor.charFormat().background() == PySide2.QtCore.Qt.red + return cursor.charFormat().background() == qtpy.QtCore.Qt.red if __name__ == '__main__': diff --git a/requirements_gui.txt b/requirements_gui.txt index c589c3517..b7ff2d6b0 100644 --- a/requirements_gui.txt +++ b/requirements_gui.txt @@ -1 +1,3 @@ PySide2==5.15.2.1 +QtPy==1.11.3;python_version<"3.7" +QtPy==2.3.0;python_version>="3.7" From a22391f518e92e54d234dce06597cca688d371a3 Mon Sep 17 00:00:00 2001 From: Nuwan Jayawardene Date: Thu, 2 Mar 2023 00:53:21 +0530 Subject: [PATCH 21/56] Update Blender version in sample Dockerfile. (#1259) --- samples/rqd/blender/Dockerfile | 34 +++++++++++++++++++ .../rqd/blender/blender2.79-docker/Dockerfile | 25 -------------- 2 files changed, 34 insertions(+), 25 deletions(-) create mode 100644 samples/rqd/blender/Dockerfile delete mode 100644 samples/rqd/blender/blender2.79-docker/Dockerfile diff --git a/samples/rqd/blender/Dockerfile b/samples/rqd/blender/Dockerfile new file mode 100644 index 000000000..208f3c7a7 --- /dev/null +++ b/samples/rqd/blender/Dockerfile @@ -0,0 +1,34 @@ +# Builds on the latest base image of RQD from Docker Hub +FROM opencue/rqd + +# Install dependencies to run Blender on the opencue/rqd image +RUN yum -y update +RUN yum -y install \ + bzip2 \ + libfreetype6 \ + libgl1-mesa-dev \ + libXi-devel \ + mesa-libGLU-devel \ + zlib-devel \ + libXinerama-devel \ + libXrandr-devel + +# Set Blender install directory +ARG BLENDER_INSTALL_DIR=/usr/local/blender + +# Set Blender download source +ARG BLENDER_DOWNLOAD_SRC=https://download.blender.org/release/Blender3.3/blender-3.3.3-linux-x64.tar.xz + +# Download and install Blender +RUN mkdir ${BLENDER_INSTALL_DIR} +RUN curl -SL ${BLENDER_DOWNLOAD_SRC} \ + -o blender.tar.xz + +RUN tar -xvf blender.tar.xz \ + -C ${BLENDER_INSTALL_DIR} \ + --strip-components=1 + +RUN rm blender.tar.xz + +# Verify Blender installation +RUN ${BLENDER_INSTALL_DIR}/blender --version diff --git a/samples/rqd/blender/blender2.79-docker/Dockerfile b/samples/rqd/blender/blender2.79-docker/Dockerfile deleted file mode 100644 index 2c1b908f7..000000000 --- a/samples/rqd/blender/blender2.79-docker/Dockerfile +++ /dev/null @@ -1,25 +0,0 @@ -# Builds on the latest base image of RQD from Docker Hub -FROM opencue/rqd - -# Install dependencies to run Blender on the opencue/rqd image -RUN yum -y update -RUN yum -y install \ - bzip2 \ - libfreetype6 \ - libgl1-mesa-dev \ - libXi-devel \ - mesa-libGLU-devel \ - zlib-devel \ - libXinerama-devel \ - libXrandr-devel - -# Download and install Blender 2.79 -RUN mkdir /usr/local/blender -RUN curl -SL https://download.blender.org/release/Blender2.79/blender-2.79-linux-glibc219-x86_64.tar.bz2 \ - -o blender.tar.bz2 - -RUN tar -jxvf blender.tar.bz2 \ - -C /usr/local/blender \ - --strip-components=1 - -RUN rm blender.tar.bz2 From deef621ecae9369d62fcefb8f3d6260284fffb36 Mon Sep 17 00:00:00 2001 From: Brian Cipriano Date: Wed, 1 Mar 2023 16:23:09 -0500 Subject: [PATCH 22/56] [pyoutline] Move outline.cfg into the outline module. (#1252) --- pyoutline/outline/config.py | 11 ++++------- pyoutline/{etc => outline}/outline.cfg | 0 pyoutline/setup.py | 8 ++++++-- pyoutline/tests/config_test.py | 2 +- 4 files changed, 11 insertions(+), 10 deletions(-) rename pyoutline/{etc => outline}/outline.cfg (100%) diff --git a/pyoutline/outline/config.py b/pyoutline/outline/config.py index de74a8faa..8cf298f62 100644 --- a/pyoutline/outline/config.py +++ b/pyoutline/outline/config.py @@ -110,13 +110,10 @@ def read_config_from_disk(): config_file = config_from_user_profile if not config_file: - default_config_paths = [__file_path__.parent.parent.parent / 'etc' / 'outline.cfg', - __file_path__.parent.parent / 'etc' / 'outline.cfg'] - for default_config_path in default_config_paths: - logger.info('Loading default outline config from %s', default_config_path) - if default_config_path.exists(): - config_file = default_config_path - break + default_config_path = __file_path__.parent / 'outline.cfg' + logger.info('Loading default outline config from %s', default_config_path) + if default_config_path.exists(): + config_file = default_config_path if not config_file: raise FileNotFoundError('outline config file was not found') diff --git a/pyoutline/etc/outline.cfg b/pyoutline/outline/outline.cfg similarity index 100% rename from pyoutline/etc/outline.cfg rename to pyoutline/outline/outline.cfg diff --git a/pyoutline/setup.py b/pyoutline/setup.py index a18b50d4f..0ecc79e0c 100644 --- a/pyoutline/setup.py +++ b/pyoutline/setup.py @@ -52,8 +52,12 @@ packages=find_packages(exclude=['tests']), data_files=[ ('bin', ['bin/cuerunbase.py', 'bin/pycuerun', 'bin/util_qc_job_layer.py']), - ('etc', ['etc/outline.cfg']), - ('wrappers', ['wrappers/opencue_wrap_frame', 'wrappers/opencue_wrap_frame_no_ss', 'wrappers/local_wrap_frame']), + ('wrappers', [ + 'wrappers/opencue_wrap_frame', 'wrappers/opencue_wrap_frame_no_ss', + 'wrappers/local_wrap_frame']), ], + package_data={ + 'outline': ['outline.cfg'], + }, test_suite='tests', ) diff --git a/pyoutline/tests/config_test.py b/pyoutline/tests/config_test.py index 5e0132d61..a7e089e39 100644 --- a/pyoutline/tests/config_test.py +++ b/pyoutline/tests/config_test.py @@ -60,7 +60,7 @@ def test__should_load_default_values(self): self.assertIsNone(os.environ.get('OL_CONF')) self.assertIsNone(os.environ.get('OUTLINE_CONFIG_FILE')) self.fs.add_real_file( - os.path.join(os.path.dirname(os.path.dirname(outline.__file__)), 'etc', 'outline.cfg'), + os.path.join(os.path.dirname(outline.__file__), 'outline.cfg'), read_only=True) config = read_config_from_disk() From 2f7f2974c4033119b61da33fb3948a40411fe7c6 Mon Sep 17 00:00:00 2001 From: Brian Cipriano Date: Wed, 1 Mar 2023 16:24:00 -0500 Subject: [PATCH 23/56] Upgrade future to 0.18.3. (#1254) --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 44adbe5b8..4ca2a8b42 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,7 @@ 2to3==1.0 enum34==1.1.6 evdev==1.4.0;python_version<"3.0" and "linux" in sys_platform -future==0.17.1 +future==0.18.3 futures==3.2.0;python_version<"3.0" grpcio==1.26.0;python_version<"3.0" grpcio-tools==1.26.0;python_version<"3.0" From 60cbecd137146900a5dcb784b5a39d95e4c6fd85 Mon Sep 17 00:00:00 2001 From: Brian Cipriano Date: Wed, 1 Mar 2023 16:25:14 -0500 Subject: [PATCH 24/56] [cuebot] Configure test logging for gradle --info and --debug. (#1263) --- cuebot/build.gradle | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/cuebot/build.gradle b/cuebot/build.gradle index a9b7cfaa5..6715a0ae6 100644 --- a/cuebot/build.gradle +++ b/cuebot/build.gradle @@ -1,4 +1,7 @@ +import org.gradle.api.tasks.testing.logging.TestExceptionFormat +import org.gradle.api.tasks.testing.logging.TestLogEvent + buildscript { repositories { mavenCentral() @@ -167,3 +170,26 @@ tasks.withType(AbstractArchiveTask) { preserveFileTimestamps = false reproducibleFileOrder = true } + +tasks.withType(Test) { + // Configure logging when running Gradle with --info or --debug. + testLogging { + info { + // Don't show STANDARD_OUT messages, these clutter up the output + // and make it hard to find actual failures. + events TestLogEvent.FAILED + exceptionFormat TestExceptionFormat.FULL + showStandardStreams false + } + debug { + // Show everything. + events TestLogEvent.STARTED, + TestLogEvent.FAILED, + TestLogEvent.PASSED, + TestLogEvent.SKIPPED, + TestLogEvent.STANDARD_ERROR, + TestLogEvent.STANDARD_OUT + exceptionFormat TestExceptionFormat.FULL + } + } +} From 2b7bfdddef7b7994a0dc5800142ab6a22f88d528 Mon Sep 17 00:00:00 2001 From: Brian Cipriano Date: Wed, 1 Mar 2023 16:45:56 -0500 Subject: [PATCH 25/56] [cuegui] Fix Comment dialog macros load and add tests. (#1261) --- cuegui/cuegui/Comments.py | 7 +++- cuegui/tests/Comments_tests.py | 74 ++++++++++++++++++++++++++++++++++ 2 files changed, 79 insertions(+), 2 deletions(-) create mode 100644 cuegui/tests/Comments_tests.py diff --git a/cuegui/cuegui/Comments.py b/cuegui/cuegui/Comments.py index c58c7bd79..9cccdf576 100644 --- a/cuegui/cuegui/Comments.py +++ b/cuegui/cuegui/Comments.py @@ -209,8 +209,11 @@ def refreshComments(self): def __macroLoad(self): """Loads the defined comment macros from settings""" - self.__macroList = pickle.loads( - str(self.app.settings.value("Comments", pickle.dumps({})))) + try: + self.__macroList = pickle.loads(self.app.settings.value("Comments", pickle.dumps({}))) + except TypeError: + self.__macroList = pickle.loads( + str(self.app.settings.value("Comments", pickle.dumps({})))) self.__macroRefresh() def __macroRefresh(self): diff --git a/cuegui/tests/Comments_tests.py b/cuegui/tests/Comments_tests.py new file mode 100644 index 000000000..29bdf613f --- /dev/null +++ b/cuegui/tests/Comments_tests.py @@ -0,0 +1,74 @@ +# Copyright (c) OpenCue Project Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +"""Tests for cuegui.Comments.""" + + +import time +import unittest + +import mock + +from qtpy import QtCore +from qtpy import QtWidgets + +import opencue.compiled_proto.comment_pb2 +import opencue.compiled_proto.job_pb2 +import opencue.wrappers.comment +import opencue.wrappers.job + +import cuegui.Comments +import cuegui.Style + +from . import test_utils + + +@mock.patch('opencue.cuebot.Cuebot.getStub', new=mock.Mock()) +class CommentsTests(unittest.TestCase): + @mock.patch('opencue.cuebot.Cuebot.getStub') + def setUp(self, getStubMock): + app = test_utils.createApplication() + app.settings = QtCore.QSettings() + cuegui.Style.init() + + commentProto = opencue.compiled_proto.comment_pb2.Comment( + id='comment-id-1', timestamp=int(time.time()), user='user-who-made-comment', + subject='comment-subject', message='this is the comment message body') + self.comment = opencue.wrappers.comment.Comment(commentProto) + getStubMock.return_value.GetComments.return_value = \ + opencue.compiled_proto.job_pb2.JobGetCommentsResponse( + comments=opencue.compiled_proto.comment_pb2.CommentSeq(comments=[commentProto])) + + self.job = opencue.wrappers.job.Job(opencue.compiled_proto.job_pb2.Job(name='fooJob')) + self.parentWidget = QtWidgets.QWidget() + self.commentListDialog = cuegui.Comments.CommentListDialog( + self.job, parent=self.parentWidget) + + def test_shouldDisplayComment(self): + self.assertEqual( + 1, self.commentListDialog._CommentListDialog__treeSubjects.topLevelItemCount()) + gotTreeWidgetItem = self.commentListDialog._CommentListDialog__treeSubjects.topLevelItem(0) + gotComment = gotTreeWidgetItem._Comment__comment + self.assertEqual(self.comment.timestamp(), gotComment.timestamp()) + self.assertEqual(self.comment.user(), gotComment.user()) + self.assertEqual(self.comment.subject(), gotComment.subject()) + self.assertEqual(self.comment.message(), gotComment.message()) + + def test_shouldRefreshJobComments(self): + self.job.getComments = mock.Mock(return_value=[]) + + self.commentListDialog.refreshComments() + + self.job.getComments.assert_called() From f74c1acf7b23c63aa68bb870fe39ac6306a44991 Mon Sep 17 00:00:00 2001 From: Brian Cipriano Date: Thu, 16 Mar 2023 13:31:21 -0400 Subject: [PATCH 26/56] Notes from Mar 1 TSC meeting. (#1272) --- tsc/meetings/2023-03-01.md | 47 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) create mode 100644 tsc/meetings/2023-03-01.md diff --git a/tsc/meetings/2023-03-01.md b/tsc/meetings/2023-03-01.md new file mode 100644 index 000000000..9c065a76c --- /dev/null +++ b/tsc/meetings/2023-03-01.md @@ -0,0 +1,47 @@ +# OpenCue TSC Meeting Notes 1 Mar 2023 + +Secretary: Brian Cipriano + +Agenda/Notes: + +* Managing database migrations + * https://github.com/AcademySoftwareFoundation/OpenCue/pull/1228 + * Sequential numbering in the repo creates problems for users with forks. Migrations in forks + need to be renumbered after the fact to not conflict, and reapplying these changes to a + production database is tricky. + * Suggestion: leave gaps in between main repo migrations, e.g. v30, v40, etc. Forks can fill + this in. + * This could also create problems if v40 conflicts with v32 for example, and cause a need to + renumber fork migrations still. + * Diego and Brian to do some further research on this. + * Another suggestion: fork migrations could use version numbers starting at a very high number + e.g. 1000. Fork migrations would always be applied on top of the full main repo schema. + * Any conflicts would need to be resolved by the user. + * Any new main repo migrations would need to be applied manually. Flyway won't apply v40 if + it thinks the database is at v1000. + * This might be the least painful option. +* Customizing frame display colors + * https://github.com/AcademySoftwareFoundation/OpenCue/pull/1246 + * Author has given more context in the PR. + * It feels wrong to store color information in the database, it's purely a visual change. + * Brian to take another look. + * Maybe we can change PR to use something like "reason code" in the database rather than color + information directly, and update cuegui.yaml to convert reason code -> color. +* Preparing for next release + * PySide6 cuegui changes + * Merged, done. + * CueSubmit PySide6 + * Not started yet. Need to include this in the same release. + * Update test script to run example job + * https://github.com/AcademySoftwareFoundation/OpenCue/pull/1253 + * Ready for review. + * Includes pycue and pyoutline example scripts which should be generally useful to users + beyond tests. + * Config guide doc + * https://github.com/AcademySoftwareFoundation/opencue.io/pull/274 + * Ready for review, to be merged/published once release is done. + * show_stats table PR + * Is there a better way to test potentially destructive changes? + * There's no easy way in pure SQL to verify changes before dropping cols/tables. + * We should expand our doc on applying database migrations to cover a db backup/restore. + * The current change seems fine, good to merge. From cd770343b9e1f6b0db9761b44e4ec95b4619f366 Mon Sep 17 00:00:00 2001 From: Brian Cipriano Date: Tue, 21 Mar 2023 12:50:26 -0400 Subject: [PATCH 27/56] Remove obsolete pyoutline subdir from Dockerfiles. (#1273) --- cuesubmit/Dockerfile | 1 - pyoutline/Dockerfile | 1 - 2 files changed, 2 deletions(-) diff --git a/cuesubmit/Dockerfile b/cuesubmit/Dockerfile index cc8f51178..6f97d671f 100644 --- a/cuesubmit/Dockerfile +++ b/cuesubmit/Dockerfile @@ -41,7 +41,6 @@ RUN 2to3 -wn -f import pycue/opencue/compiled_proto/*_pb2*.py COPY pyoutline/README.md ./pyoutline/ COPY pyoutline/setup.py ./pyoutline/ COPY pyoutline/bin ./pyoutline/bin -COPY pyoutline/etc ./pyoutline/etc COPY pyoutline/wrappers ./pyoutline/wrappers COPY pyoutline/outline ./pyoutline/outline diff --git a/pyoutline/Dockerfile b/pyoutline/Dockerfile index e898452e1..02f954c23 100644 --- a/pyoutline/Dockerfile +++ b/pyoutline/Dockerfile @@ -29,7 +29,6 @@ RUN 2to3 -wn -f import pycue/opencue/compiled_proto/*_pb2*.py COPY pyoutline/README.md ./pyoutline/ COPY pyoutline/setup.py ./pyoutline/ COPY pyoutline/bin ./pyoutline/bin -COPY pyoutline/etc ./pyoutline/etc COPY pyoutline/tests/ ./pyoutline/tests COPY pyoutline/wrappers ./pyoutline/wrappers COPY pyoutline/outline ./pyoutline/outline From d091d56c5a66cd5dbb4d2ca17d0a1696e1161b60 Mon Sep 17 00:00:00 2001 From: Brian Cipriano Date: Tue, 21 Mar 2023 13:11:45 -0400 Subject: [PATCH 28/56] [ci] Update test script to run a job and confirm it finishes. (#1253) --- .github/workflows/temp-integration-test.yml | 25 ++++++++ ci/run_integration_test.sh | 30 ++++++++- samples/outline-files/hello.outline | 6 -- samples/outline-files/hello_frame.outline | 6 -- samples/pycue/wait_for_job.py | 61 +++++++++++++++++++ .../hellomodule.py => pyoutline/basic_job.py} | 21 +++++-- .../example_module.py} | 0 7 files changed, 130 insertions(+), 19 deletions(-) create mode 100644 .github/workflows/temp-integration-test.yml delete mode 100644 samples/outline-files/hello.outline delete mode 100644 samples/outline-files/hello_frame.outline create mode 100755 samples/pycue/wait_for_job.py rename samples/{outline-modules/hellomodule.py => pyoutline/basic_job.py} (56%) mode change 100644 => 100755 rename samples/{outline-modules/example.py => pyoutline/example_module.py} (100%) diff --git a/.github/workflows/temp-integration-test.yml b/.github/workflows/temp-integration-test.yml new file mode 100644 index 000000000..3a746276d --- /dev/null +++ b/.github/workflows/temp-integration-test.yml @@ -0,0 +1,25 @@ +name: (TEMP) OpenCue Integration Test + +on: + push: + branches: [ master ] + pull_request: + branches: [ master ] + +jobs: + integration_test: + name: Run Integration Test + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v3 + + - name: Run test + run: ci/run_integration_test.sh + + - name: Archive log files + uses: actions/upload-artifact@v3 + if: ${{ always() }} + with: + name: test-logs + path: /tmp/opencue-test/*.log diff --git a/ci/run_integration_test.sh b/ci/run_integration_test.sh index ee0a37fe6..8bde5335d 100755 --- a/ci/run_integration_test.sh +++ b/ci/run_integration_test.sh @@ -1,4 +1,25 @@ #!/bin/bash +# Copyright Contributors to the OpenCue Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# OpenCue integration test script +# +# Stands up a clean environment using Docker compose and verifies all +# components are functioning as expected. +# +# Run with: +# ./run_integration_test.sh set -e @@ -162,6 +183,13 @@ test_cueadmin() { fi } +run_job() { + samples/pyoutline/basic_job.py + job_name="testing-shot01-${USER}_basic_job" + samples/pycue/wait_for_job.py "${job_name}" --timeout 300 + log INFO "Job succeeded (PASS)" +} + cleanup() { docker compose rm --stop --force >>"${DOCKER_COMPOSE_LOG}" 2>&1 rm -rf "${RQD_ROOT}" || true @@ -224,7 +252,7 @@ main() { log INFO "Testing cueadmin..." test_cueadmin - # TODO Launch a job and verify it finishes. + run_job cleanup diff --git a/samples/outline-files/hello.outline b/samples/outline-files/hello.outline deleted file mode 100644 index 6288ca30c..000000000 --- a/samples/outline-files/hello.outline +++ /dev/null @@ -1,6 +0,0 @@ -#!/usr/bin/env python - -from outline.modules.shell import Shell - -Shell("hello", - command="echo 'Hello World!'") diff --git a/samples/outline-files/hello_frame.outline b/samples/outline-files/hello_frame.outline deleted file mode 100644 index c060719e1..000000000 --- a/samples/outline-files/hello_frame.outline +++ /dev/null @@ -1,6 +0,0 @@ -#!/usr/bin/env python - -from outline.modules.shell import Shell - -Shell("hello", - command="echo 'Hello frame %{FRAME}!'") diff --git a/samples/pycue/wait_for_job.py b/samples/pycue/wait_for_job.py new file mode 100755 index 000000000..d66de1a44 --- /dev/null +++ b/samples/pycue/wait_for_job.py @@ -0,0 +1,61 @@ +#!/usr/bin/env python +# Copyright Contributors to the OpenCue Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Basic script that waits for a job to complete.""" + +import argparse +import datetime +import logging +import sys +import time + +import opencue +from opencue.wrappers.job import Job + + +def wait_for_job(job_name, timeout_sec=None): + logging.basicConfig( + level=logging.INFO, + format='%(asctime)s %(levelname)s %(message)s', + datefmt='%Y-%m-%d %H:%M:%S') + logging.info('Waiting for job %s...', job_name) + start_time = datetime.datetime.now() + while True: + if (datetime.datetime.now() - start_time).seconds > timeout_sec: + logging.error('Timed out') + return False + jobs = opencue.api.getJobs(job=[job_name], include_finished=True) + if not jobs: + logging.error("Job %s not found", job_name) + return False + job = jobs[0] + logging.info('Job state = %s', Job.JobState(job.state()).name) + if job.state() == Job.JobState.FINISHED: + logging.info('Job succeeded') + return True + if job.deadFrames() > 0: + logging.error('Job is failing') + return False + time.sleep(5) + + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument("job_name", help="name of the job to wait for") + parser.add_argument("--timeout", help="number of seconds to wait before timing out", type=int) + args = parser.parse_args() + result = wait_for_job(args.job_name, timeout_sec=args.timeout) + if not result: + sys.exit(1) diff --git a/samples/outline-modules/hellomodule.py b/samples/pyoutline/basic_job.py old mode 100644 new mode 100755 similarity index 56% rename from samples/outline-modules/hellomodule.py rename to samples/pyoutline/basic_job.py index 2e8f09ae1..7cae6eb19 --- a/samples/outline-modules/hellomodule.py +++ b/samples/pyoutline/basic_job.py @@ -1,5 +1,4 @@ #!/usr/bin/env python - # Copyright Contributors to the OpenCue Project # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -14,11 +13,21 @@ # See the License for the specific language governing permissions and # limitations under the License. +"""Basic job structure with a single layer and five frames. + +The frames just print out the current frame number.""" + + +import getpass -from outline import Outline, cuerun -from outline.modules.tutorial import HelloModule +import outline +import outline.cuerun +import outline.modules.shell -ol = Outline("my_job") -ol.add_layer(HelloModule("my_layer")) -cuerun.launch(ol, range="1-10", pause=True) +ol = outline.Outline( + 'basic_job', shot='shot01', show='testing', user=getpass.getuser()) +layer = outline.modules.shell.Shell( + 'echo_frame', command=['echo', '#IFRAME#'], chunk=1, threads=1, range='1-5') +ol.add_layer(layer) +outline.cuerun.launch(ol, use_pycuerun=False) diff --git a/samples/outline-modules/example.py b/samples/pyoutline/example_module.py similarity index 100% rename from samples/outline-modules/example.py rename to samples/pyoutline/example_module.py From 669d6c401034b560fb698b547efffa902ab6444c Mon Sep 17 00:00:00 2001 From: Brian Cipriano Date: Tue, 21 Mar 2023 13:27:58 -0400 Subject: [PATCH 29/56] Remove temp workflow. (#1274) --- .github/workflows/temp-integration-test.yml | 25 --------------------- 1 file changed, 25 deletions(-) delete mode 100644 .github/workflows/temp-integration-test.yml diff --git a/.github/workflows/temp-integration-test.yml b/.github/workflows/temp-integration-test.yml deleted file mode 100644 index 3a746276d..000000000 --- a/.github/workflows/temp-integration-test.yml +++ /dev/null @@ -1,25 +0,0 @@ -name: (TEMP) OpenCue Integration Test - -on: - push: - branches: [ master ] - pull_request: - branches: [ master ] - -jobs: - integration_test: - name: Run Integration Test - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v3 - - - name: Run test - run: ci/run_integration_test.sh - - - name: Archive log files - uses: actions/upload-artifact@v3 - if: ${{ always() }} - with: - name: test-logs - path: /tmp/opencue-test/*.log From 6fa72ffba969510a109daf75abe3cca9b72bcf6e Mon Sep 17 00:00:00 2001 From: Brian Cipriano Date: Tue, 30 May 2023 14:02:14 -0400 Subject: [PATCH 30/56] Notes from TSC meeting. (#1294) --- tsc/meetings/2023-04-12.md | 38 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) create mode 100644 tsc/meetings/2023-04-12.md diff --git a/tsc/meetings/2023-04-12.md b/tsc/meetings/2023-04-12.md new file mode 100644 index 000000000..43d001d65 --- /dev/null +++ b/tsc/meetings/2023-04-12.md @@ -0,0 +1,38 @@ +# OpenCue TSC Meeting Notes 12 Apr 2023 + +Secretary: Brian Cipriano + +Agenda/Notes: + +* Host ping time until marked as DOWN + * https://github.com/AcademySoftwareFoundation/OpenCue/issues/1265 + * This does seem long, any reason why? + * Diego: seems unusual. Should be less than a minute for host to be marked DOWN. CueGUI should + update 15-20s + * Should we make this a config setting? + * SPI to check on their code for differences + * Might need to lower default value, this is a good candidate for config flag. +* RQD config file overhaul + * https://github.com/AcademySoftwareFoundation/OpenCue/pull/1270 + * Would be good to get SPI input on this. + * Let's review in more detail. No immediate concerns. SPI has some similar configuration already +* Rez setup email thread + * https://lists.aswf.io/g/opencue-dev/topic/97805737#571 + * Diego: might make a better tutorial doc than merging into master branch. We don't want to + confuse new users with multiple packaging options. + * Look into spk, an OSS project. + * pip packages will make this setup much simpler. +* Prepending timestamps to RQD child process output + * https://github.com/AcademySoftwareFoundation/OpenCue/pull/1286 + * Doesn't modify any existing output other than prepending timestamp to each line. + * Linux-specific, configurable. +* Python 2 support + * Not ready to drop support for 2 entirely, especially python libraries. + * GUI should be fine to go 3-only. + * If we're going to do it, document which tag contains the last 2 support + * A py2 branch might be helpful if anyone wants to backport, but might have issues with our + versioning tooling. +* Blender plugin update + * Basic plugin is loading, currently navigating issues with installing pyoutline into Blender + environment. Will start to send test jobs soon + * Will continue to update email thread. From 48abfd819f0278d15d38f3f9f458a35a940cc66a Mon Sep 17 00:00:00 2001 From: Diego Tavares da Silva Date: Wed, 7 Jun 2023 14:10:29 -0700 Subject: [PATCH 31/56] Prepend every rqd log line with a timestamp (#1286) * Prepend every rqd log line with a timestamp Not all applications launched on rqd have logs with a timestamp, which makes is difficult to debug jobs that are taking more than expected on the cue. This feature prepends a timestamp for every line. * Add rqconstant to turn timestamp feature on/off * Add rqconstant to turn timestamp feature on/off --- rqd/rqd/rqconstants.py | 3 ++ rqd/rqd/rqcore.py | 108 ++++++++++++++++++++++++++++++++++++-- rqd/tests/rqcore_tests.py | 3 +- 3 files changed, 108 insertions(+), 6 deletions(-) diff --git a/rqd/rqd/rqconstants.py b/rqd/rqd/rqconstants.py index ad82b5daf..fe793970e 100644 --- a/rqd/rqd/rqconstants.py +++ b/rqd/rqd/rqconstants.py @@ -73,6 +73,7 @@ RQD_BECOME_JOB_USER = True RQD_CREATE_USER_IF_NOT_EXISTS = True RQD_TAGS = '' +RQD_PREPEND_TIMESTAMP = False KILL_SIGNAL = 9 if platform.system() == 'Linux': @@ -197,6 +198,8 @@ if config.has_option(__section, "FILE_LOG_LEVEL"): level = config.get(__section, "FILE_LOG_LEVEL") FILE_LOG_LEVEL = logging.getLevelName(level) + if config.has_option(__section, "RQD_PREPEND_TIMESTAMP"): + RQD_PREPEND_TIMESTAMP = config.getboolean(__section, "RQD_PREPEND_TIMESTAMP") # pylint: disable=broad-except except Exception as e: logging.warning( diff --git a/rqd/rqd/rqcore.py b/rqd/rqd/rqcore.py index 224485f2b..ce073ac45 100644 --- a/rqd/rqd/rqcore.py +++ b/rqd/rqd/rqcore.py @@ -315,14 +315,17 @@ def runLinux(self): else: tempCommand += [self._createCommandFile(runFrame.command)] - # Actual cwd is set by /shots/SHOW/home/perl/etc/qwrap.cuerun + if rqd.rqconstants.RQD_PREPEND_TIMESTAMP: + file_descriptor = subprocess.PIPE + else: + file_descriptor = self.rqlog # pylint: disable=subprocess-popen-preexec-fn frameInfo.forkedCommand = subprocess.Popen(tempCommand, env=self.frameEnv, cwd=self.rqCore.machine.getTempPath(), stdin=subprocess.PIPE, - stdout=self.rqlog, - stderr=self.rqlog, + stdout=file_descriptor, + stderr=file_descriptor, close_fds=True, preexec_fn=os.setsid) finally: @@ -335,6 +338,8 @@ def runLinux(self): self.rqCore.updateRss) self.rqCore.updateRssThread.start() + if rqd.rqconstants.RQD_PREPEND_TIMESTAMP: + pipe_to_file(frameInfo.forkedCommand.stdout, frameInfo.forkedCommand.stderr, self.rqlog) returncode = frameInfo.forkedCommand.wait() # Find exitStatus and exitSignal @@ -535,7 +540,7 @@ def run(self): else: raise RuntimeError(err) try: - self.rqlog = open(runFrame.log_dir_file, "w", 1) + self.rqlog = open(runFrame.log_dir_file, "w+", 1) self.waitForFile(runFrame.log_dir_file) # pylint: disable=broad-except except Exception as e: @@ -1161,3 +1166,98 @@ def sendStatusReport(self): def isWaitingForIdle(self): """Returns whether the host is waiting until idle to take some action.""" return self.__whenIdle + +def pipe_to_file(stdout, stderr, outfile): + """ + Prepend entries on stdout and stderr with a timestamp and write to outfile. + + The logic to poll stdout/stderr is inspired by the Popen.communicate implementation. + This feature is linux specific + """ + # Importing packages internally to avoid compatibility issues with Windows + + if stdout is None or stderr is None: + return + outfile.flush() + os.fsync(outfile) + + import select + import errno + + fd2file = {} + fd2output = {} + + poller = select.poll() + + def register_and_append(file_ojb, eventmask): + poller.register(file_ojb, eventmask) + fd2file[file_ojb.fileno()] = file_ojb + + def close_and_unregister_and_remove(fd, close=False): + poller.unregister(fd) + if close: + fd2file[fd].close() + fd2file.pop(fd) + + def print_and_flush_ln(fd, last_timestamp): + txt = ''.join(fd2output[fd]) + lines = txt.split('\n') + next_line_timestamp = None + + # Save the timestamp of the first break + if last_timestamp is None: + curr_line_timestamp = datetime.datetime.now().strftime("%H:%M:%S") + else: + curr_line_timestamp = last_timestamp + + # There are no line breaks + if len(lines) < 2: + return curr_line_timestamp + else: + next_line_timestamp = datetime.datetime.now().strftime("%H:%M:%S") + + remainder = lines[-1] + for line in lines[0:-1]: + print("[%s] %s" % (curr_line_timestamp, line), file=outfile) + outfile.flush() + os.fsync(outfile) + fd2output[fd] = [remainder] + + if next_line_timestamp is None: + return curr_line_timestamp + else: + return next_line_timestamp + + def translate_newlines(data): + data = data.decode("utf-8", "ignore") + return data.replace("\r\n", "\n").replace("\r", "\n") + + select_POLLIN_POLLPRI = select.POLLIN | select.POLLPRI + # stdout + register_and_append(stdout, select_POLLIN_POLLPRI) + fd2output[stdout.fileno()] = [] + + # stderr + register_and_append(stderr, select_POLLIN_POLLPRI) + fd2output[stderr.fileno()] = [] + + while fd2file: + try: + ready = poller.poll() + except select.error as e: + if e.args[0] == errno.EINTR: + continue + raise + + first_chunk_timestamp = None + for fd, mode in ready: + if mode & select_POLLIN_POLLPRI: + data = os.read(fd, 4096) + if not data: + close_and_unregister_and_remove(fd) + if not isinstance(data, str): + data = translate_newlines(data) + fd2output[fd].append(data) + first_chunk_timestamp = print_and_flush_ln(fd, first_chunk_timestamp) + else: + close_and_unregister_and_remove(fd) diff --git a/rqd/tests/rqcore_tests.py b/rqd/tests/rqcore_tests.py index aad307dac..256b96b2b 100644 --- a/rqd/tests/rqcore_tests.py +++ b/rqd/tests/rqcore_tests.py @@ -567,6 +567,7 @@ def setUp(self): @mock.patch('platform.system', new=mock.Mock(return_value='Linux')) @mock.patch('tempfile.gettempdir') + @mock.patch('rqd.rqcore.pipe_to_file', new=mock.MagicMock()) def test_runLinux(self, getTempDirMock, permsUser, timeMock, popenMock): # mkdirMock, openMock, # given currentTime = 1568070634.3 @@ -632,8 +633,6 @@ def test_runLinux(self, getTempDirMock, permsUser, timeMock, popenMock): # mkdir self.assertTrue(os.path.exists(logDir)) self.assertTrue(os.path.isfile(logFile)) _, kwargs = popenMock.call_args - self.assertEqual(logFile, kwargs['stdout'].name) - self.assertEqual(logFile, kwargs['stderr'].name) rqCore.network.reportRunningFrameCompletion.assert_called_with( rqd.compiled_proto.report_pb2.FrameCompleteReport( From 0199fb2445f03cb700c77eb11515e935479d124d Mon Sep 17 00:00:00 2001 From: Brian Cipriano Date: Wed, 14 Jun 2023 17:53:52 -0400 Subject: [PATCH 32/56] Notes from June 7 TSC meeting. (#1298) --- tsc/meetings/2023-06-07.md | 44 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) create mode 100644 tsc/meetings/2023-06-07.md diff --git a/tsc/meetings/2023-06-07.md b/tsc/meetings/2023-06-07.md new file mode 100644 index 000000000..98df98a61 --- /dev/null +++ b/tsc/meetings/2023-06-07.md @@ -0,0 +1,44 @@ +# OpenCue TSC Meeting Notes 7 Jun 2023 + +Secretary: Brian Cipriano + +Agenda/Notes: + +* Host ping time until marked as DOWN + * https://github.com/AcademySoftwareFoundation/OpenCue/issues/1265 + * Any update here? + * Needs some further verification and response. +* Appending timestamps to logs + * https://github.com/AcademySoftwareFoundation/OpenCue/pull/1286 + * LGTM, needs merge from master, looking into test failures. +* Cuesubmit batch of PRs + * https://github.com/AcademySoftwareFoundation/OpenCue/pull/1278 + * https://github.com/AcademySoftwareFoundation/OpenCue/pull/1280 + * https://github.com/AcademySoftwareFoundation/OpenCue/pull/1281 + * https://github.com/AcademySoftwareFoundation/OpenCue/pull/1282 + * https://github.com/AcademySoftwareFoundation/OpenCue/pull/1283 + * https://github.com/AcademySoftwareFoundation/OpenCue/pull/1284 + * https://github.com/AcademySoftwareFoundation/OpenCue/pull/1285 + * Reviews needed. + * Be careful we're not making CueSubmit too specialized, keep it generally useful for new + users. + * Let's invite the author to a TSC meeting soon. + * Improvements are good, is there something else we can offer? What would be helpful for + larger-studio users? Or is the Python library good enough? + * Best to expand pyoutline examples / docs to help developers who have already tried + CueSubmit. + * Build on basic example used in integration test script. +* Blender plugin update + * Currently testing job submission, blocked on some submission code. + * Loading python deps (opencue, filesequence) + * Can manually copy into blender plugin directory, but how to automate this? + * Does Blender offer alternatives e.g. configuring plugin path via env var? + * Look into creating additional packages, maybe as empty packages. +* Openshift Cuebot version + * Putting multiple Cuebots behind gRPC load balancer, and pointing RQD at the LB. Currently to + take a Cuebot offline all RQDs need to be restarted to move to a new Cuebot host, this solves + that problem. + * Would make a good tutorial or sample to include in the main repo. + * Prometheus export needs to be reworked. Currently using a separate client to query metrics, + which doesn't work with the LB setup as it will not redirect requests to a consistent Cuebot. + Working on a change to send metrics directly from Cuebot. From d2057d3f1c3c01d5a78fedd9a22fe8400d91ec83 Mon Sep 17 00:00:00 2001 From: Diego Tavares da Silva Date: Fri, 16 Jun 2023 09:57:07 -0700 Subject: [PATCH 33/56] Migrate stats columns from show to show_stats (#1228) * Migrate stats columns from show to show_stats Frequently changing columns on the SHOW table were causing the show indexes to bloat faster than desired and impacting the system performance, specially in the Whiteboard queries for jobs. * Fix merge issues Add gpu columns * Version bump * Update cuebot/src/main/java/com/imageworks/spcue/dao/postgres/WhiteboardDaoJdbc.java Signed-off-by: Diego Tavares da Silva * Creating a new show should also create a show_stats entry A show_stats row needs to be created and deleted together with a show row. This bug was introduced on https://gitlab.spimageworks.com/spi/dev/infrastructure/api/opencue/-/merge_requests/427 * Rename V21__AddShowStats.sql to V16__AddShowStats.sql --------- Signed-off-by: Diego Tavares da Silva --- .../spcue/dao/postgres/JobDaoJdbc.java | 2 +- .../spcue/dao/postgres/ShowDaoJdbc.java | 10 +++++- .../spcue/dao/postgres/WhiteboardDaoJdbc.java | 17 +++++++++- .../postgres/migrations/V16__AddShowStats.sql | 33 +++++++++++++++++++ .../resources/conf/ddl/postgres/seed_data.sql | 4 ++- .../spcue/test/dao/postgres/ShowDaoTests.java | 8 ++--- .../resources/conf/ddl/postgres/test_data.sql | 8 +++-- 7 files changed, 72 insertions(+), 10 deletions(-) create mode 100644 cuebot/src/main/resources/conf/ddl/postgres/migrations/V16__AddShowStats.sql diff --git a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/JobDaoJdbc.java b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/JobDaoJdbc.java index d32a5a259..a5f595f4e 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/JobDaoJdbc.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/JobDaoJdbc.java @@ -574,7 +574,7 @@ public void activateJob(JobInterface job, JobState jobState) { jobTotals[0] + jobTotals[1], layers.size(), job.getJobId()); getJdbcTemplate().update( - "UPDATE show SET int_frame_insert_count=int_frame_insert_count+?, int_job_insert_count=int_job_insert_count+1 WHERE pk_show=?", + "UPDATE show_stats SET int_frame_insert_count=int_frame_insert_count+?, int_job_insert_count=int_job_insert_count+1 WHERE pk_show=?", jobTotals[0] + jobTotals[1], job.getShowId()); updateState(job, jobState); diff --git a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/ShowDaoJdbc.java b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/ShowDaoJdbc.java index 8361a6c5f..86e126559 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/ShowDaoJdbc.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/ShowDaoJdbc.java @@ -138,9 +138,15 @@ public ShowEntity getShowDetail(HostInterface host) { private static final String INSERT_SHOW = "INSERT INTO show (pk_show,str_name) VALUES (?,?)"; + private static final String INSERT_SHOW_STATS = + "INSERT INTO show_stats " + + "(pk_show, int_frame_insert_count, int_job_insert_count, int_frame_success_count, int_frame_fail_count) " + + "VALUES (?, 0, 0, 0, 0)"; + public void insertShow(ShowEntity show) { show.id = SqlUtil.genKeyRandom(); getJdbcTemplate().update(INSERT_SHOW, show.id, show.name); + getJdbcTemplate().update(INSERT_SHOW_STATS, show.id); } private static final String SHOW_EXISTS = @@ -169,6 +175,8 @@ public void delete(ShowInterface s) { s.getShowId()); getJdbcTemplate().update("DELETE FROM show_alias WHERE pk_show=?", s.getShowId()); + getJdbcTemplate().update("DELETE FROM show_stats WHERE pk_show=?", + s.getShowId()); getJdbcTemplate().update("DELETE FROM show WHERE pk_show=?", s.getShowId()); } @@ -262,7 +270,7 @@ public void updateFrameCounters(ShowInterface s, int exitStatus) { col = "int_frame_fail_count = int_frame_fail_count + 1"; } getJdbcTemplate().update( - "UPDATE show SET " + col + " WHERE pk_show=?", s.getShowId()); + "UPDATE show_stats SET " + col + " WHERE pk_show=?", s.getShowId()); } } diff --git a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/WhiteboardDaoJdbc.java b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/WhiteboardDaoJdbc.java index 38566470a..fe3217f96 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/WhiteboardDaoJdbc.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/WhiteboardDaoJdbc.java @@ -2060,7 +2060,21 @@ public Show mapRow(ResultSet rs, int rowNum) throws SQLException { private static final String GET_SHOW = "SELECT " + - "show.*," + + "show.pk_show," + + "show.str_name," + + "show.b_paused," + + "show.int_default_min_cores," + + "show.int_default_max_cores," + + "show.int_default_min_gpus," + + "show.int_default_max_gpus," + + "show.b_booking_enabled," + + "show.b_dispatch_enabled," + + "show.b_active," + + "show.str_comment_email," + + "show_stats.int_frame_insert_count," + + "show_stats.int_job_insert_count," + + "show_stats.int_frame_success_count," + + "show_stats.int_frame_fail_count," + "COALESCE(vs_show_stat.int_pending_count,0) AS int_pending_count," + "COALESCE(vs_show_stat.int_running_count,0) AS int_running_count," + "COALESCE(vs_show_stat.int_dead_count,0) AS int_dead_count," + @@ -2069,6 +2083,7 @@ public Show mapRow(ResultSet rs, int rowNum) throws SQLException { "COALESCE(vs_show_stat.int_job_count,0) AS int_job_count " + "FROM " + "show " + + "JOIN show_stats ON (show.pk_show = show_stats.pk_show) " + "LEFT JOIN vs_show_stat ON (vs_show_stat.pk_show = show.pk_show) " + "LEFT JOIN vs_show_resource ON (vs_show_resource.pk_show=show.pk_show) " + "WHERE " + diff --git a/cuebot/src/main/resources/conf/ddl/postgres/migrations/V16__AddShowStats.sql b/cuebot/src/main/resources/conf/ddl/postgres/migrations/V16__AddShowStats.sql new file mode 100644 index 000000000..f97c14f7c --- /dev/null +++ b/cuebot/src/main/resources/conf/ddl/postgres/migrations/V16__AddShowStats.sql @@ -0,0 +1,33 @@ +CREATE TABLE show_stats ( + pk_show VARCHAR(36) NOT NULL, + int_frame_insert_count BIGINT DEFAULT 0 NOT NULL, + int_job_insert_count BIGINT DEFAULT 0 NOT NULL, + int_frame_success_count BIGINT DEFAULT 0 NOT NULL, + int_frame_fail_count BIGINT DEFAULT 0 NOT NULL +); + +INSERT INTO show_stats ( + pk_show, + int_frame_insert_count, + int_job_insert_count, + int_frame_success_count, + int_frame_fail_count +) SELECT + pk_show, + int_frame_insert_count, + int_job_insert_count, + int_frame_success_count, + int_frame_fail_count + FROM show; + +CREATE UNIQUE INDEX c_show_stats_pk ON show_stats (pk_show); +ALTER TABLE show_stats ADD CONSTRAINT c_show_stats_pk PRIMARY KEY + USING INDEX c_show_stats_pk; + + +-- Destructive changes. Please test changes above prior to executing this. +ALTER TABLE show + DROP COLUMN int_frame_insert_count, + DROP COLUMN int_job_insert_count, + DROP COLUMN int_frame_success_count, + DROP COLUMN int_frame_fail_count; diff --git a/cuebot/src/main/resources/conf/ddl/postgres/seed_data.sql b/cuebot/src/main/resources/conf/ddl/postgres/seed_data.sql index ebebaa75d..f5707b62c 100644 --- a/cuebot/src/main/resources/conf/ddl/postgres/seed_data.sql +++ b/cuebot/src/main/resources/conf/ddl/postgres/seed_data.sql @@ -1,4 +1,6 @@ -Insert into SHOW (PK_SHOW,STR_NAME,INT_DEFAULT_MAX_CORES,INT_DEFAULT_MIN_CORES,INT_FRAME_INSERT_COUNT,INT_JOB_INSERT_COUNT,INT_FRAME_SUCCESS_COUNT,INT_FRAME_FAIL_COUNT,B_BOOKING_ENABLED,B_DISPATCH_ENABLED,B_ACTIVE) values ('00000000-0000-0000-0000-000000000000', 'testing', 200000, 100, 0, 0, 0, 0, true, true, true); +Insert into SHOW (PK_SHOW,STR_NAME,INT_DEFAULT_MAX_CORES,INT_DEFAULT_MIN_CORES,B_BOOKING_ENABLED,B_DISPATCH_ENABLED,B_ACTIVE) values ('00000000-0000-0000-0000-000000000000', 'testing', 200000, 100, true, true, true); + +Insert into SHOW_STATS (PK_SHOW,INT_FRAME_INSERT_COUNT,INT_JOB_INSERT_COUNT,INT_FRAME_SUCCESS_COUNT,INT_FRAME_FAIL_COUNT) values ('00000000-0000-0000-0000-000000000000',0,0,0,0) Insert into SHOW_ALIAS (PK_SHOW_ALIAS,PK_SHOW,STR_NAME) values ('00000000-0000-0000-0000-000000000001', '00000000-0000-0000-0000-000000000000', 'test'); diff --git a/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/ShowDaoTests.java b/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/ShowDaoTests.java index d430ab3b0..e44393ab6 100644 --- a/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/ShowDaoTests.java +++ b/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/ShowDaoTests.java @@ -219,20 +219,20 @@ public void testUpdateActive() { public void testUpdateFrameCounters() { ShowEntity show = showDao.findShowDetail(SHOW_NAME); int frameSuccess = jdbcTemplate.queryForObject( - "SELECT int_frame_success_count FROM show WHERE pk_show=?", + "SELECT int_frame_success_count FROM show_stats WHERE pk_show=?", Integer.class, show.id); showDao.updateFrameCounters(show, 0); int frameSucces2 = jdbcTemplate.queryForObject( - "SELECT int_frame_success_count FROM show WHERE pk_show=?", + "SELECT int_frame_success_count FROM show_stats WHERE pk_show=?", Integer.class, show.id); assertEquals(frameSuccess + 1,frameSucces2); int frameFail= jdbcTemplate.queryForObject( - "SELECT int_frame_fail_count FROM show WHERE pk_show=?", + "SELECT int_frame_fail_count FROM show_stats WHERE pk_show=?", Integer.class, show.id); showDao.updateFrameCounters(show, 1); int frameFail2 = jdbcTemplate.queryForObject( - "SELECT int_frame_fail_count FROM show WHERE pk_show=?", + "SELECT int_frame_fail_count FROM show_stats WHERE pk_show=?", Integer.class, show.id); assertEquals(frameFail+ 1,frameFail2); } diff --git a/cuebot/src/test/resources/conf/ddl/postgres/test_data.sql b/cuebot/src/test/resources/conf/ddl/postgres/test_data.sql index ae7b77354..4f9c2a0a0 100644 --- a/cuebot/src/test/resources/conf/ddl/postgres/test_data.sql +++ b/cuebot/src/test/resources/conf/ddl/postgres/test_data.sql @@ -1,6 +1,10 @@ -Insert into SHOW (PK_SHOW,STR_NAME,INT_DEFAULT_MAX_CORES,INT_DEFAULT_MIN_CORES,INT_FRAME_INSERT_COUNT,INT_JOB_INSERT_COUNT,INT_FRAME_SUCCESS_COUNT,INT_FRAME_FAIL_COUNT,B_BOOKING_ENABLED,B_DISPATCH_ENABLED,B_ACTIVE) values ('00000000-0000-0000-0000-000000000000','pipe',20000,100,0,0,0,0,true,true,true) +Insert into SHOW (PK_SHOW,STR_NAME,INT_DEFAULT_MAX_CORES,INT_DEFAULT_MIN_CORES,B_BOOKING_ENABLED,B_DISPATCH_ENABLED,B_ACTIVE) values ('00000000-0000-0000-0000-000000000000','pipe',20000,100,true,true,true) -Insert into SHOW (PK_SHOW,STR_NAME,INT_DEFAULT_MAX_CORES,INT_DEFAULT_MIN_CORES,INT_FRAME_INSERT_COUNT,INT_JOB_INSERT_COUNT,INT_FRAME_SUCCESS_COUNT,INT_FRAME_FAIL_COUNT,B_BOOKING_ENABLED,B_DISPATCH_ENABLED,B_ACTIVE) values ('00000000-0000-0000-0000-000000000001','edu',20000,100,0,0,0,0,true,true,true) +Insert into SHOW (PK_SHOW,STR_NAME,INT_DEFAULT_MAX_CORES,INT_DEFAULT_MIN_CORES,B_BOOKING_ENABLED,B_DISPATCH_ENABLED,B_ACTIVE) values ('00000000-0000-0000-0000-000000000001','edu',20000,100,true,true,true) + +Insert into SHOW_STATS (PK_SHOW,INT_FRAME_INSERT_COUNT,INT_JOB_INSERT_COUNT,INT_FRAME_SUCCESS_COUNT,INT_FRAME_FAIL_COUNT) values ('00000000-0000-0000-0000-000000000000',0,0,0,0) + +Insert into SHOW_STATS (PK_SHOW,INT_FRAME_INSERT_COUNT,INT_JOB_INSERT_COUNT,INT_FRAME_SUCCESS_COUNT,INT_FRAME_FAIL_COUNT) values ('00000000-0000-0000-0000-000000000001',0,0,0,0) Insert into SHOW_ALIAS (PK_SHOW_ALIAS,PK_SHOW,STR_NAME) values ('00000000-0000-0000-0000-000000000001','00000000-0000-0000-0000-000000000000','fx') From 6703e07de6982c7372647d2aa0216b86979b320a Mon Sep 17 00:00:00 2001 From: Brian Cipriano Date: Thu, 13 Jul 2023 14:03:37 -0400 Subject: [PATCH 34/56] Test pipeline fixes and refresh. (#1305) --- .github/workflows/testing-pipeline.yml | 67 +++++-------------- ci/run_python_tests.sh | 8 ++- ci/run_python_tests_pyside6.sh | 48 +++++++++++++ .../resources/conf/ddl/postgres/seed_data.sql | 2 +- cuegui/tests/Constants_tests.py | 7 +- requirements.txt | 3 +- rqd/rqd/rqcore.py | 8 +-- 7 files changed, 85 insertions(+), 58 deletions(-) create mode 100755 ci/run_python_tests_pyside6.sh diff --git a/.github/workflows/testing-pipeline.yml b/.github/workflows/testing-pipeline.yml index 7d2b59af9..15967a4a9 100644 --- a/.github/workflows/testing-pipeline.yml +++ b/.github/workflows/testing-pipeline.yml @@ -7,41 +7,20 @@ on: branches: [ master ] jobs: - test_python_2019: - name: Run Python Unit Tests (CY2019) - runs-on: ubuntu-latest - container: aswf/ci-opencue:2019 - steps: - - uses: actions/checkout@v3 - - name: Run Python Tests - run: ci/run_python_tests.sh - - test_cuebot_2019: - name: Build Cuebot and Run Unit Tests (CY2019) - runs-on: ubuntu-latest - container: - image: aswf/ci-opencue:2019 - steps: - - uses: actions/checkout@v3 - - name: Build with Gradle - run: | - chown -R aswfuser:aswfgroup . - su -c "cd cuebot && ./gradlew build --stacktrace --info" aswfuser - - test_python_2020: - name: Run Python Unit Tests (CY2020) + test_python_2022: + name: Run Python Unit Tests (CY2022) runs-on: ubuntu-latest - container: aswf/ci-opencue:2020 + container: aswf/ci-opencue:2022 steps: - uses: actions/checkout@v3 - name: Run Python Tests - run: ci/run_python_tests.sh + run: ci/run_python_tests.sh --no-gui - test_cuebot_2020: - name: Build Cuebot and Run Unit Tests (CY2020) + test_cuebot_2022: + name: Build Cuebot and Run Unit Tests (CY2022) runs-on: ubuntu-latest container: - image: aswf/ci-opencue:2020 + image: aswf/ci-opencue:2022 steps: - uses: actions/checkout@v3 - name: Build with Gradle @@ -49,20 +28,20 @@ jobs: chown -R aswfuser:aswfgroup . su -c "cd cuebot && ./gradlew build --stacktrace --info" aswfuser - test_python_2021: - name: Run Python Unit Tests (CY2021) + test_python_2023: + name: Run Python Unit Tests (CY2023) runs-on: ubuntu-latest - container: aswf/ci-opencue:2021 + container: aswf/ci-opencue:2023 steps: - uses: actions/checkout@v3 - name: Run Python Tests run: ci/run_python_tests.sh - test_cuebot_2021: - name: Build Cuebot and Run Unit Tests (CY2021) + test_cuebot_2023: + name: Build Cuebot and Run Unit Tests (CY2023) runs-on: ubuntu-latest container: - image: aswf/ci-opencue:2021 + image: aswf/ci-opencue:2023 steps: - uses: actions/checkout@v3 - name: Build with Gradle @@ -70,27 +49,15 @@ jobs: chown -R aswfuser:aswfgroup . su -c "cd cuebot && ./gradlew build --stacktrace --info" aswfuser - test_python_2022: - name: Run Python Unit Tests (CY2022) + test_python2: + name: Run Python Unit Tests using Python2 runs-on: ubuntu-latest - container: aswf/ci-opencue:2022 + container: aswf/ci-opencue:2019 steps: - uses: actions/checkout@v3 - name: Run Python Tests run: ci/run_python_tests.sh - test_cuebot_2022: - name: Build Cuebot and Run Unit Tests (CY2022) - runs-on: ubuntu-latest - container: - image: aswf/ci-opencue:2022 - steps: - - uses: actions/checkout@v3 - - name: Build with Gradle - run: | - chown -R aswfuser:aswfgroup . - su -c "cd cuebot && ./gradlew build --stacktrace --info" aswfuser - test_pyside6: name: Run CueGUI Tests using PySide6 runs-on: ubuntu-latest @@ -113,7 +80,7 @@ jobs: name: Test Documentation Build runs-on: ubuntu-latest container: - image: aswf/ci-opencue:2020 + image: aswf/ci-opencue:2023 steps: - uses: actions/checkout@v3 - name: Run Sphinx build diff --git a/ci/run_python_tests.sh b/ci/run_python_tests.sh index c6a51a015..5f1bfe294 100755 --- a/ci/run_python_tests.sh +++ b/ci/run_python_tests.sh @@ -1,7 +1,13 @@ #!/bin/bash +# Script for running OpenCue unit tests with PySide2. +# +# This script is written to be run within the OpenCue GitHub Actions environment. +# See `.github/workflows/testing-pipeline.yml`. + set -e +args=("$@") python_version=$(python -V 2>&1) echo "Will run tests using ${python_version}" @@ -23,6 +29,6 @@ PYTHONPATH=pycue:pyoutline python cuesubmit/setup.py test python rqd/setup.py test # Xvfb no longer supports Python 2. -if [[ "$python_version" =~ "Python 3" ]]; then +if [[ "$python_version" =~ "Python 3" && ${args[0]} != "--no-gui" ]]; then ci/run_gui_test.sh fi diff --git a/ci/run_python_tests_pyside6.sh b/ci/run_python_tests_pyside6.sh new file mode 100755 index 000000000..384841cfe --- /dev/null +++ b/ci/run_python_tests_pyside6.sh @@ -0,0 +1,48 @@ +#!/bin/bash + +# Script for running OpenCue unit tests with PySide6. +# +# This script is written to be run within the OpenCue GitHub Actions environment. +# See `.github/workflows/testing-pipeline.yml`. + +set -e + +python_version=$(python -V 2>&1) +echo "Will run tests using ${python_version}" + +# NOTE: To run this in an almalinux environment, install these packages: +# yum -y install \ +# dbus-libs \ +# fontconfig \ +# gcc \ +# libxkbcommon-x11 \ +# mesa-libEGL-devel \ +# python-devel \ +# which \ +# xcb-util-keysyms \ +# xcb-util-image \ +# xcb-util-renderutil \ +# xcb-util-wm \ +# Xvfb + +# Install Python requirements. +python3 -m pip install --user -r requirements.txt -r requirements_gui.txt +# Replace PySide2 with PySide6. +python3 -m pip uninstall -y PySide2 +python3 -m pip install --user PySide6==6.3.2 + +# Protos need to have their Python code generated in order for tests to pass. +python -m grpc_tools.protoc -I=proto/ --python_out=pycue/opencue/compiled_proto --grpc_python_out=pycue/opencue/compiled_proto proto/*.proto +python -m grpc_tools.protoc -I=proto/ --python_out=rqd/rqd/compiled_proto --grpc_python_out=rqd/rqd/compiled_proto proto/*.proto + +# Fix compiled proto code for Python 3. +2to3 -wn -f import pycue/opencue/compiled_proto/*_pb2*.py +2to3 -wn -f import rqd/rqd/compiled_proto/*_pb2*.py + +python pycue/setup.py test +PYTHONPATH=pycue python pyoutline/setup.py test +PYTHONPATH=pycue python cueadmin/setup.py test +PYTHONPATH=pycue:pyoutline python cuesubmit/setup.py test +python rqd/setup.py test + +ci/run_gui_test.sh diff --git a/cuebot/src/main/resources/conf/ddl/postgres/seed_data.sql b/cuebot/src/main/resources/conf/ddl/postgres/seed_data.sql index f5707b62c..7b189174c 100644 --- a/cuebot/src/main/resources/conf/ddl/postgres/seed_data.sql +++ b/cuebot/src/main/resources/conf/ddl/postgres/seed_data.sql @@ -1,6 +1,6 @@ Insert into SHOW (PK_SHOW,STR_NAME,INT_DEFAULT_MAX_CORES,INT_DEFAULT_MIN_CORES,B_BOOKING_ENABLED,B_DISPATCH_ENABLED,B_ACTIVE) values ('00000000-0000-0000-0000-000000000000', 'testing', 200000, 100, true, true, true); -Insert into SHOW_STATS (PK_SHOW,INT_FRAME_INSERT_COUNT,INT_JOB_INSERT_COUNT,INT_FRAME_SUCCESS_COUNT,INT_FRAME_FAIL_COUNT) values ('00000000-0000-0000-0000-000000000000',0,0,0,0) +Insert into SHOW_STATS (PK_SHOW,INT_FRAME_INSERT_COUNT,INT_JOB_INSERT_COUNT,INT_FRAME_SUCCESS_COUNT,INT_FRAME_FAIL_COUNT) values ('00000000-0000-0000-0000-000000000000',0,0,0,0); Insert into SHOW_ALIAS (PK_SHOW_ALIAS,PK_SHOW,STR_NAME) values ('00000000-0000-0000-0000-000000000001', '00000000-0000-0000-0000-000000000000', 'test'); diff --git a/cuegui/tests/Constants_tests.py b/cuegui/tests/Constants_tests.py index 9466dcece..3cfe3866f 100644 --- a/cuegui/tests/Constants_tests.py +++ b/cuegui/tests/Constants_tests.py @@ -25,7 +25,7 @@ import mock import pyfakefs.fake_filesystem_unittest -from PySide2 import QtGui +from qtpy import QtGui import opencue import cuegui.Constants @@ -40,6 +40,7 @@ ''' +# pylint: disable=import-outside-toplevel,redefined-outer-name,reimported class ConstantsTests(pyfakefs.fake_filesystem_unittest.TestCase): def setUp(self): self.setUpPyfakefs() @@ -53,6 +54,7 @@ def test__should_load_user_config_from_env_var(self): self.fs.create_file(config_file_path, contents=CONFIG_YAML) os.environ['CUEGUI_CONFIG_FILE'] = config_file_path + import cuegui.Constants result = importlib.reload(cuegui.Constants) self.assertEqual('98.707.68', result.VERSION) @@ -65,6 +67,7 @@ def test__should_load_user_config_from_user_profile(self): config_file_path = '/home/username/.config/opencue/cuegui.yaml' self.fs.create_file(config_file_path, contents=CONFIG_YAML) + import cuegui.Constants result = importlib.reload(cuegui.Constants) self.assertEqual('98.707.68', result.VERSION) @@ -73,6 +76,7 @@ def test__should_load_user_config_from_user_profile(self): @mock.patch('platform.system', new=mock.Mock(return_value='Linux')) def test__should_use_default_values(self): + import cuegui.Constants result = importlib.reload(cuegui.Constants) self.assertNotEqual('98.707.68', result.VERSION) @@ -161,6 +165,7 @@ def test__should_use_default_values(self): @mock.patch('platform.system', new=mock.Mock(return_value='Darwin')) def test__should_use_mac_editor(self): + import cuegui.Constants result = importlib.reload(cuegui.Constants) self.assertEqual('open -t', result.DEFAULT_EDITOR) diff --git a/requirements.txt b/requirements.txt index 4ca2a8b42..262f681f9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -12,7 +12,8 @@ packaging==20.9 pathlib==1.0.1;python_version<"3.4" protobuf==3.17.3;python_version<"3.0" psutil==5.6.7 -pyfakefs==3.6 +pyfakefs==3.6;python_version<"3.7" +pyfakefs==5.2.3;python_version>="3.7" pylint==2.6.0;python_version>="3.7" pynput==1.7.6 PyYAML==5.1 diff --git a/rqd/rqd/rqcore.py b/rqd/rqd/rqcore.py index ce073ac45..842028732 100644 --- a/rqd/rqd/rqcore.py +++ b/rqd/rqd/rqcore.py @@ -1181,8 +1181,10 @@ def pipe_to_file(stdout, stderr, outfile): outfile.flush() os.fsync(outfile) + # pylint: disable=import-outside-toplevel import select import errno + # pylint: enable=import-outside-toplevel fd2file = {} fd2output = {} @@ -1213,8 +1215,7 @@ def print_and_flush_ln(fd, last_timestamp): # There are no line breaks if len(lines) < 2: return curr_line_timestamp - else: - next_line_timestamp = datetime.datetime.now().strftime("%H:%M:%S") + next_line_timestamp = datetime.datetime.now().strftime("%H:%M:%S") remainder = lines[-1] for line in lines[0:-1]: @@ -1225,8 +1226,7 @@ def print_and_flush_ln(fd, last_timestamp): if next_line_timestamp is None: return curr_line_timestamp - else: - return next_line_timestamp + return next_line_timestamp def translate_newlines(data): data = data.decode("utf-8", "ignore") From c1f335d22e59cdf75859aa14ecdfe43d9cb43e95 Mon Sep 17 00:00:00 2001 From: Kern Attila GERMAIN <5556461+KernAttila@users.noreply.github.com> Date: Wed, 9 Aug 2023 17:47:45 +0200 Subject: [PATCH 35/56] [cuesubmit] Add tooltip to display available command tokens. (#1278) --- .../spcue/dispatcher/DispatchSupportService.java | 4 ++++ cuesubmit/cuesubmit/Constants.py | 13 +++++++++++++ cuesubmit/cuesubmit/ui/Command.py | 8 ++++---- 3 files changed, 21 insertions(+), 4 deletions(-) diff --git a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/DispatchSupportService.java b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/DispatchSupportService.java index c935024d6..887d0c29e 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/DispatchSupportService.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/DispatchSupportService.java @@ -404,6 +404,10 @@ public RunFrame prepareRqdRunFrame(VirtualProc proc, DispatchFrame frame) { .replaceAll("#JOB#", frame.jobName) .replaceAll("#FRAMESPEC#", frameSpec) .replaceAll("#FRAME#", frame.name)); + /* The special command tokens above (#ZFRAME# and others) are provided to the user in cuesubmit. + * see: cuesubmit/cuesubmit/Constants.py + * Update the Constant.py file when updating tokens here, they will appear in the cuesubmit tooltip popup. + */ frame.uid.ifPresent(builder::setUid); diff --git a/cuesubmit/cuesubmit/Constants.py b/cuesubmit/cuesubmit/Constants.py index 123f32723..b4f82f13f 100644 --- a/cuesubmit/cuesubmit/Constants.py +++ b/cuesubmit/cuesubmit/Constants.py @@ -39,6 +39,19 @@ BLENDER_RENDER_CMD = config.get('BLENDER_RENDER_CMD', 'blender') FRAME_TOKEN = config.get('FRAME_TOKEN', '#IFRAME#') +# Tokens are replaced by cuebot during dispatch with their computed value. +# see: cuebot/src/main/java/com/imageworks/spcue/dispatcher/DispatchSupportService.java +# Update this file when updating tokens in cuebot, they will appear in the cuesubmit tooltip popup. +COMMAND_TOKENS = {'#ZFRAME#': 'Current frame with a padding of 4', + '#IFRAME#': 'Current frame', + '#FRAME_START#': 'First frame of chunk', + '#FRAME_END#': 'Last frame of chunk', + '#FRAME_CHUNK#': 'Chunk size', + '#FRAMESPEC#': 'Full frame range', + '#LAYER#': 'Name of the Layer', + '#JOB#': 'Name of the Job', + '#FRAME#': 'Name of the Frame' + } BLENDER_FORMATS = ['', 'AVIJPEG', 'AVIRAW', 'BMP', 'CINEON', 'DPX', 'EXR', 'HDR', 'IRIS', 'IRIZ', 'JP2', 'JPEG', 'MPEG', 'MULTILAYER', 'PNG', 'RAWTGA', 'TGA', 'TIFF'] BLENDER_OUTPUT_OPTIONS_URL = \ diff --git a/cuesubmit/cuesubmit/ui/Command.py b/cuesubmit/cuesubmit/ui/Command.py index 548e2a836..c1d144b24 100644 --- a/cuesubmit/cuesubmit/ui/Command.py +++ b/cuesubmit/cuesubmit/ui/Command.py @@ -23,6 +23,7 @@ from PySide2 import QtCore, QtWidgets from cuesubmit.ui import Widgets +from cuesubmit import Constants class CueCommandWidget(Widgets.CueHelpWidget): @@ -69,11 +70,10 @@ def __init__(self, *args, **kwargs): self.commandBox.setAccessibleName('commandBox') self.horizontalLine = Widgets.CueHLine() self.setFixedHeight(120) + tokensToolTip = '\n'.join([' {0} -- {1}'.format(token, info) + for token, info in Constants.COMMAND_TOKENS.items()]) self.commandBox.setToolTip('Enter the command to be run. Valid replacement tokens are:\n' - ' #IFRAME# -- frame number\n' - ' #LAYER# -- layer name\n' - ' #JOB# -- job name\n' - ' #FRAME# -- frame name') + + tokensToolTip) self.setupUi() def setupUi(self): From baa122a033420258d9488dacc4364da6d4c42ddb Mon Sep 17 00:00:00 2001 From: Jimmy Christensen Date: Wed, 27 Sep 2023 22:26:44 +0200 Subject: [PATCH 36/56] Fix reading stats from /proc/{pid}/statm (#1308) --- rqd/rqd/rqmachine.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/rqd/rqd/rqmachine.py b/rqd/rqd/rqmachine.py index 2fc5a32ce..65061ef82 100644 --- a/rqd/rqd/rqmachine.py +++ b/rqd/rqd/rqmachine.py @@ -216,7 +216,9 @@ def __updateGpuAndLlu(self, frame): def _getStatFields(self, pidFilePath): with open(pidFilePath, "r") as statFile: - return [None, None] + statFile.read().rsplit(")", 1)[-1].split() + stats = statFile.read().split() + stats[1] = stats[1].strip('()') + return stats def rssUpdate(self, frames): """Updates the rss and maxrss for all running frames""" From b606a593a2ab26f3b26caf806a1ce397c1491a40 Mon Sep 17 00:00:00 2001 From: Ramon Figueiredo Date: Wed, 27 Sep 2023 15:38:33 -0700 Subject: [PATCH 37/56] [cuebot] Prevent booking frames on hosts with no temp space. (#1306) --- .../com/imageworks/spcue/dao/CommentDao.java | 22 ++ .../com/imageworks/spcue/dao/HostDao.java | 8 + .../spcue/dao/postgres/CommentDaoJdbc.java | 13 + .../spcue/dao/postgres/HostDaoJdbc.java | 7 + .../spcue/dispatcher/HostReportHandler.java | 79 +++++- .../spcue/service/CommentManager.java | 22 ++ .../spcue/service/CommentManagerService.java | 12 + .../imageworks/spcue/service/HostManager.java | 8 + .../spcue/service/HostManagerService.java | 5 + cuebot/src/main/resources/opencue.properties | 6 + .../test/dao/criteria/ProcSearchTests.java | 6 +- .../test/dao/postgres/BookingDaoTests.java | 5 +- .../test/dao/postgres/CommentDaoTests.java | 5 +- .../spcue/test/dao/postgres/DeedDaoTests.java | 5 +- .../dao/postgres/DispatcherDaoFifoTests.java | 6 +- .../test/dao/postgres/DispatcherDaoTests.java | 5 +- .../test/dao/postgres/FrameDaoTests.java | 5 +- .../spcue/test/dao/postgres/HostDaoTests.java | 5 +- .../spcue/test/dao/postgres/ProcDaoTests.java | 5 +- .../spcue/test/dao/postgres/ShowDaoTests.java | 5 +- .../test/dao/postgres/WhiteboardDaoTests.java | 5 +- .../CoreUnitDispatcherGpuJobTests.java | 5 +- .../CoreUnitDispatcherGpuTests.java | 5 +- .../CoreUnitDispatcherGpusJobTests.java | 5 +- .../dispatcher/CoreUnitDispatcherTests.java | 6 +- .../test/dispatcher/DispatchSupportTests.java | 5 +- .../dispatcher/FrameCompleteHandlerTests.java | 10 +- .../test/dispatcher/HistoryControlTests.java | 5 +- .../dispatcher/HostReportHandlerGpuTests.java | 5 +- .../dispatcher/HostReportHandlerTests.java | 237 ++++++++++++++++-- .../test/dispatcher/LocalDispatcherTests.java | 5 +- .../test/dispatcher/RedirectManagerTests.java | 6 +- .../test/dispatcher/StrandedCoreTests.java | 6 +- .../test/dispatcher/TestBookingQueue.java | 5 +- .../test/service/BookingManagerTests.java | 5 +- .../spcue/test/service/HostManagerTests.java | 5 +- .../spcue/test/service/JobManagerTests.java | 5 +- .../spcue/test/service/OwnerManagerTests.java | 5 +- cuebot/src/test/resources/opencue.properties | 6 + 39 files changed, 489 insertions(+), 81 deletions(-) diff --git a/cuebot/src/main/java/com/imageworks/spcue/dao/CommentDao.java b/cuebot/src/main/java/com/imageworks/spcue/dao/CommentDao.java index 5568bb2fd..08bea59b1 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dao/CommentDao.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dao/CommentDao.java @@ -23,6 +23,8 @@ import com.imageworks.spcue.HostInterface; import com.imageworks.spcue.JobInterface; +import java.util.List; + public interface CommentDao { /** @@ -32,6 +34,26 @@ public interface CommentDao { */ public void deleteComment(String id); + /** + * Deletes comments using host, user, and subject + * + * @param host + * @param user + * @param subject + * @return boolean: returns true if one or more comments where deleted + */ + public boolean deleteCommentByHostUserAndSubject(HostInterface host, String user, String subject); + + /** + * Get comments using host, user, and subject + * + * @param host + * @param user + * @param subject + * @return List + */ + public List getCommentsByHostUserAndSubject(HostInterface host, String user, String subject); + /** * Retrieves the specified comment. * diff --git a/cuebot/src/main/java/com/imageworks/spcue/dao/HostDao.java b/cuebot/src/main/java/com/imageworks/spcue/dao/HostDao.java index 768bcdbd2..5ed18947e 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dao/HostDao.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dao/HostDao.java @@ -78,6 +78,14 @@ public interface HostDao { */ void updateHostState(HostInterface host, HardwareState state); + /** + * updates a host with the passed free temporary directory + * + * @param host + * @param freeTempDir + */ + void updateHostFreeTempDir(HostInterface host, Long freeTempDir); + /** * returns a full host detail * diff --git a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/CommentDaoJdbc.java b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/CommentDaoJdbc.java index 9587e41db..ea61f07bb 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/CommentDaoJdbc.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/CommentDaoJdbc.java @@ -21,6 +21,7 @@ import java.sql.ResultSet; import java.sql.SQLException; +import java.util.List; import java.util.Map; import org.springframework.jdbc.core.RowMapper; @@ -71,6 +72,18 @@ public CommentDetail mapRow(ResultSet rs, int row) throws SQLException { } }; + public boolean deleteCommentByHostUserAndSubject(HostInterface host, String user, String subject) { + return getJdbcTemplate().update( + "DELETE FROM comments WHERE pk_host=? AND str_user=? AND str_subject=?", + host.getHostId(), user, subject) > 0; + } + + public List getCommentsByHostUserAndSubject(HostInterface host, String user, String subject) { + return getJdbcTemplate().query( + "SELECT * FROM comments WHERE pk_host=? AND str_user=? AND str_subject=?", + COMMENT_DETAIL_MAPPER, host.getHostId(), user, subject); + } + public CommentDetail getCommentDetail(String id) { return getJdbcTemplate().queryForObject( "SELECT * FROM comments WHERE pk_comment=?", diff --git a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/HostDaoJdbc.java b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/HostDaoJdbc.java index 5c106335c..6fe898b44 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/HostDaoJdbc.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/HostDaoJdbc.java @@ -523,6 +523,13 @@ public void updateHostState(HostInterface host, HardwareState state) { state.toString(), host.getHostId()); } + @Override + public void updateHostFreeTempDir(HostInterface host, Long freeTempDir) { + getJdbcTemplate().update( + "UPDATE host_stat SET int_mcp_free=? WHERE pk_host=?", + freeTempDir, host.getHostId()); + } + @Override public void updateHostSetAllocation(HostInterface host, AllocationInterface alloc) { diff --git a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/HostReportHandler.java b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/HostReportHandler.java index d763cce53..2adef34fb 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/HostReportHandler.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/HostReportHandler.java @@ -29,10 +29,13 @@ import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.LogManager; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.core.env.Environment; import org.springframework.core.task.TaskRejectedException; import org.springframework.dao.DataAccessException; import org.springframework.dao.EmptyResultDataAccessException; +import com.imageworks.spcue.CommentDetail; import com.imageworks.spcue.DispatchHost; import com.imageworks.spcue.FrameInterface; import com.imageworks.spcue.JobEntity; @@ -57,6 +60,7 @@ import com.imageworks.spcue.rqd.RqdClient; import com.imageworks.spcue.rqd.RqdClientException; import com.imageworks.spcue.service.BookingManager; +import com.imageworks.spcue.service.CommentManager; import com.imageworks.spcue.service.HostManager; import com.imageworks.spcue.service.JobManager; import com.imageworks.spcue.service.JobManagerSupport; @@ -80,6 +84,14 @@ public class HostReportHandler { private JobManagerSupport jobManagerSupport; private JobDao jobDao; private LayerDao layerDao; + @Autowired + private Environment env; + @Autowired + private CommentManager commentManager; + // Comment constants + private static final String SUBJECT_COMMENT_FULL_TEMP_DIR = "Host set to REPAIR for not having enough storage " + + "space on the temporary directory (mcp)"; + private static final String CUEBOT_COMMENT_USER = "cuebot"; /** * Boolean to toggle if this class is accepting data or not. @@ -156,7 +168,7 @@ public void handleHostReport(HostReport report, boolean isBoot) { rhost.getLoad(), new Timestamp(rhost.getBootTime() * 1000l), rhost.getAttributesMap().get("SP_OS")); - changeHardwareState(host, report.getHost().getState(), isBoot); + changeHardwareState(host, report.getHost().getState(), isBoot, report.getHost().getFreeMcp()); changeNimbyState(host, report.getHost()); /** @@ -221,7 +233,14 @@ public void handleHostReport(HostReport report, boolean isBoot) { } } - if (host.idleCores < Dispatcher.CORE_POINTS_RESERVED_MIN) { + // The minimum amount of free space in the temporary directory to book a host + Long minBookableFreeTempDir = env.getRequiredProperty("dispatcher.min_bookable_free_temp_dir_kb", Long.class); + + if (minBookableFreeTempDir != -1 && report.getHost().getFreeMcp() < minBookableFreeTempDir) { + msg = String.format("%s doens't have enough free space in the temporary directory (mcp), %dMB needs %dMB", + host.name, (report.getHost().getFreeMcp()/1024), (minBookableFreeTempDir/1024)); + } + else if (host.idleCores < Dispatcher.CORE_POINTS_RESERVED_MIN) { msg = String.format("%s doesn't have enough idle cores, %d needs %d", host.name, host.idleCores, Dispatcher.CORE_POINTS_RESERVED_MIN); } @@ -231,7 +250,7 @@ else if (host.idleMemory < Dispatcher.MEM_RESERVED_MIN) { } else if (report.getHost().getFreeMem() < CueUtil.MB512) { msg = String.format("%s doens't have enough free system mem, %d needs %d", - host.name, report.getHost().getFreeMem(), Dispatcher.MEM_RESERVED_MIN); + host.name, report.getHost().getFreeMem(), Dispatcher.MEM_RESERVED_MIN); } else if(!host.hardwareState.equals(HardwareState.UP)) { msg = host + " is not in the Up state."; @@ -309,13 +328,61 @@ else if (!dispatchSupport.isCueBookable(host)) { * updated with a boot report. If the state is Repair, then state is * never updated via RQD. * + * + * Prevent cue frames from booking on hosts with full temporary directories. + * + * Change host state to REPAIR or UP according the amount of free space + * in the temporary directory: + * - Set the host state to REPAIR, when the amount of free space in the + * temporary directory is less than the minimum required. Add a comment with + * subject: SUBJECT_COMMENT_FULL_TEMP_DIR + * - Set the host state to UP, when the amount of free space in the temporary directory + * is greater or equals to the minimum required and the host has a comment with + * subject: SUBJECT_COMMENT_FULL_TEMP_DIR + * * @param host * @param reportState * @param isBoot + * @param freeTempDir */ - private void changeHardwareState(DispatchHost host, - HardwareState reportState, boolean isBoot) { + private void changeHardwareState(DispatchHost host, HardwareState reportState, boolean isBoot, long freeTempDir) { + + // The minimum amount of free space in the temporary directory to book a host + Long minBookableFreeTempDir = env.getRequiredProperty("dispatcher.min_bookable_free_temp_dir_kb", Long.class); + + // Prevent cue frames from booking on hosts with full temporary directories + if (minBookableFreeTempDir != -1) { + if (host.hardwareState == HardwareState.UP && freeTempDir < minBookableFreeTempDir) { + + // Insert a comment indicating that the Host status = Repair with reason = Full temporary directory + CommentDetail c = new CommentDetail(); + c.subject = SUBJECT_COMMENT_FULL_TEMP_DIR; + c.user = CUEBOT_COMMENT_USER; + c.timestamp = null; + c.message = "Host " + host.getName() + " marked as REPAIR. The current amount of free space in the " + + "temporary directory (mcp) is " + (freeTempDir/1024) + "MB. It must have at least " + + (minBookableFreeTempDir/1024) + "MB of free space in temporary directory"; + commentManager.addComment(host, c); + // Set the host state to REPAIR + hostManager.setHostState(host, HardwareState.REPAIR); + host.hardwareState = HardwareState.REPAIR; + + return; + } else if (host.hardwareState == HardwareState.REPAIR && freeTempDir >= minBookableFreeTempDir) { + // Check if the host with REPAIR status has comments with subject=SUBJECT_COMMENT_FULL_TEMP_DIR and + // user=CUEBOT_COMMENT_USER and delete the comments, if they exists + boolean commentsDeleted = commentManager.deleteCommentByHostUserAndSubject(host, + CUEBOT_COMMENT_USER, SUBJECT_COMMENT_FULL_TEMP_DIR); + + if (commentsDeleted) { + // Set the host state to UP + hostManager.setHostState(host, HardwareState.UP); + host.hardwareState = HardwareState.UP; + return; + } + } + } // If the states are the same there is no reason to do this update. if (host.hardwareState.equals(reportState)) { @@ -374,7 +441,7 @@ private void changeNimbyState(DispatchHost host, RenderHost rh) { * locked if all cores are locked. * * @param host DispatchHost - * @param renderHost RenderHost + * @param coreInfo CoreDetail */ private void changeLockState(DispatchHost host, CoreDetail coreInfo) { if (host.lockState == LockState.LOCKED) { diff --git a/cuebot/src/main/java/com/imageworks/spcue/service/CommentManager.java b/cuebot/src/main/java/com/imageworks/spcue/service/CommentManager.java index 10533c542..faee9dff9 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/service/CommentManager.java +++ b/cuebot/src/main/java/com/imageworks/spcue/service/CommentManager.java @@ -23,6 +23,8 @@ import com.imageworks.spcue.HostInterface; import com.imageworks.spcue.JobInterface; +import java.util.List; + public interface CommentManager { /** @@ -47,6 +49,26 @@ public interface CommentManager { */ public void deleteComment(String id); + /** + * Deletes comments using host, user, and subject + * + * @param host + * @param user + * @param subject + * @return boolean: returns true if one or more comments where deleted + */ + public boolean deleteCommentByHostUserAndSubject(HostInterface host, String user, String subject); + + /** + * Get comments using host, user, and subject + * + * @param host + * @param user + * @param subject + * @return List + */ + public List getCommentsByHostUserAndSubject(HostInterface host, String user, String subject); + /** * * @param id diff --git a/cuebot/src/main/java/com/imageworks/spcue/service/CommentManagerService.java b/cuebot/src/main/java/com/imageworks/spcue/service/CommentManagerService.java index cc9a016ef..b6d4430ec 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/service/CommentManagerService.java +++ b/cuebot/src/main/java/com/imageworks/spcue/service/CommentManagerService.java @@ -28,6 +28,8 @@ import com.imageworks.spcue.ShowEntity; import com.imageworks.spcue.dao.CommentDao; +import java.util.List; + @Transactional public class CommentManagerService implements CommentManager { @@ -55,6 +57,16 @@ public void deleteComment(String id) { commentDao.deleteComment(id); } + @Transactional(propagation = Propagation.REQUIRED) + public boolean deleteCommentByHostUserAndSubject(HostInterface host, String user, String subject) { + return commentDao.deleteCommentByHostUserAndSubject(host, user, subject); + } + + @Transactional(propagation = Propagation.REQUIRED) + public List getCommentsByHostUserAndSubject(HostInterface host, String user, String subject) { + return commentDao.getCommentsByHostUserAndSubject(host, user, subject); + } + @Transactional(propagation = Propagation.REQUIRED) public void setCommentSubject(String id, String subject) { commentDao.updateCommentSubject(id, subject); diff --git a/cuebot/src/main/java/com/imageworks/spcue/service/HostManager.java b/cuebot/src/main/java/com/imageworks/spcue/service/HostManager.java index 8b176c77e..aaf401688 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/service/HostManager.java +++ b/cuebot/src/main/java/com/imageworks/spcue/service/HostManager.java @@ -62,6 +62,14 @@ public interface HostManager { */ void setHostState(HostInterface host, HardwareState state); + /** + * Updates the free temporary directory (mcp) of a host. + * + * @param host HostInterface + * @param freeTempDir Long + */ + void setHostFreeTempDir(HostInterface host, Long freeTempDir); + /** * Return true if the host is swapping hard enough * that killing frames will save the entire machine. diff --git a/cuebot/src/main/java/com/imageworks/spcue/service/HostManagerService.java b/cuebot/src/main/java/com/imageworks/spcue/service/HostManagerService.java index a7c5b0729..a1533d695 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/service/HostManagerService.java +++ b/cuebot/src/main/java/com/imageworks/spcue/service/HostManagerService.java @@ -93,6 +93,11 @@ public void setHostState(HostInterface host, HardwareState state) { hostDao.updateHostState(host, state); } + @Override + public void setHostFreeTempDir(HostInterface host, Long freeTempDir) { + hostDao.updateHostFreeTempDir(host, freeTempDir); + } + @Override @Transactional(propagation = Propagation.REQUIRED, readOnly=true) public boolean isSwapping(HostInterface host) { diff --git a/cuebot/src/main/resources/opencue.properties b/cuebot/src/main/resources/opencue.properties index a08522eb1..6b2875899 100644 --- a/cuebot/src/main/resources/opencue.properties +++ b/cuebot/src/main/resources/opencue.properties @@ -110,6 +110,12 @@ dispatcher.report_queue.max_pool_size=8 # Queue capacity for handling Host Report. dispatcher.report_queue.queue_capacity=1000 +# The minimum amount of free space in the temporary directory (mcp) to book a host. +# E.g: 1G = 1048576 kB => dispatcher.min_bookable_free_temp_dir_kb=1048576 +# Default = -1 (deactivated) +# If equals to -1, it means the feature is turned off +dispatcher.min_bookable_free_temp_dir_kb=-1 + # Number of threads to keep in the pool for kill frame operation. dispatcher.kill_queue.core_pool_size=6 # Maximum number of threads to allow in the pool for kill frame operation. diff --git a/cuebot/src/test/java/com/imageworks/spcue/test/dao/criteria/ProcSearchTests.java b/cuebot/src/test/java/com/imageworks/spcue/test/dao/criteria/ProcSearchTests.java index 59f00df9a..78a13b321 100644 --- a/cuebot/src/test/java/com/imageworks/spcue/test/dao/criteria/ProcSearchTests.java +++ b/cuebot/src/test/java/com/imageworks/spcue/test/dao/criteria/ProcSearchTests.java @@ -50,6 +50,7 @@ import com.imageworks.spcue.service.HostManager; import com.imageworks.spcue.service.JobLauncher; import com.imageworks.spcue.service.JobManager; +import com.imageworks.spcue.util.CueUtil; import static org.assertj.core.api.Assertions.assertThat; import static org.junit.Assert.assertEquals; @@ -209,11 +210,12 @@ private void launchJobs() { private RenderHost.Builder buildRenderHost() { return RenderHost.newBuilder() .setBootTime(1192369572) - .setFreeMcp(76020) + // The minimum amount of free space in the temporary directory to book a host. + .setFreeMcp(CueUtil.GB) .setFreeMem(53500) .setFreeSwap(20760) .setLoad(1) - .setTotalMcp(195430) + .setTotalMcp(CueUtil.GB4) .setTotalMem(8173264) .setTotalSwap(20960) .setNimbyEnabled(false) diff --git a/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/BookingDaoTests.java b/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/BookingDaoTests.java index c6c03d604..577b53eac 100644 --- a/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/BookingDaoTests.java +++ b/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/BookingDaoTests.java @@ -96,11 +96,12 @@ public DispatchHost createHost() { RenderHost host = RenderHost.newBuilder() .setName("test_host") .setBootTime(1192369572) - .setFreeMcp(76020) + // The minimum amount of free space in the temporary directory to book a host. + .setFreeMcp(CueUtil.GB) .setFreeMem(53500) .setFreeSwap(20760) .setLoad(1) - .setTotalMcp(195430) + .setTotalMcp(CueUtil.GB4) .setTotalMem((int) CueUtil.GB16) .setTotalSwap((int) CueUtil.GB16) .setNimbyEnabled(false) diff --git a/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/CommentDaoTests.java b/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/CommentDaoTests.java index 668e666e9..9282d7b79 100644 --- a/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/CommentDaoTests.java +++ b/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/CommentDaoTests.java @@ -140,11 +140,12 @@ public void testInsertCommentOnHost() { RenderHost host = RenderHost.newBuilder() .setName("boo") .setBootTime(1192369572) - .setFreeMcp(76020) + // The minimum amount of free space in the temporary directory to book a host. + .setFreeMcp(CueUtil.GB) .setFreeMem(15290520) .setFreeSwap(2076) .setLoad(1) - .setTotalMcp(19543) + .setTotalMcp(CueUtil.GB4) .setTotalMem(15290520) .setTotalSwap(2096) .setNimbyEnabled(false) diff --git a/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/DeedDaoTests.java b/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/DeedDaoTests.java index a04e7e5e6..962b669bb 100644 --- a/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/DeedDaoTests.java +++ b/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/DeedDaoTests.java @@ -73,11 +73,12 @@ public DispatchHost createHost() { RenderHost host = RenderHost.newBuilder() .setName("test_host") .setBootTime(1192369572) - .setFreeMcp(76020) + // The minimum amount of free space in the temporary directory to book a host. + .setFreeMcp(CueUtil.GB) .setFreeMem(15290520) .setFreeSwap(2076) .setLoad(1) - .setTotalMcp(19543) + .setTotalMcp(CueUtil.GB4) .setTotalMem((int) CueUtil.GB16) .setTotalSwap((int) CueUtil.GB16) .setNimbyEnabled(false) diff --git a/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/DispatcherDaoFifoTests.java b/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/DispatcherDaoFifoTests.java index c34396709..4f6db1072 100644 --- a/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/DispatcherDaoFifoTests.java +++ b/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/DispatcherDaoFifoTests.java @@ -56,6 +56,7 @@ import com.imageworks.spcue.service.JobLauncher; import com.imageworks.spcue.service.JobManager; import com.imageworks.spcue.test.AssumingPostgresEngine; +import com.imageworks.spcue.util.CueUtil; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; @@ -147,11 +148,12 @@ public void createHost() { RenderHost host = RenderHost.newBuilder() .setName(HOSTNAME) .setBootTime(1192369572) - .setFreeMcp(76020) + // The minimum amount of free space in the temporary directory to book a host. + .setFreeMcp(CueUtil.GB) .setFreeMem(53500) .setFreeSwap(20760) .setLoad(1) - .setTotalMcp(195430) + .setTotalMcp(CueUtil.GB4) .setTotalMem(8173264) .setTotalSwap(20960) .setNimbyEnabled(false) diff --git a/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/DispatcherDaoTests.java b/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/DispatcherDaoTests.java index 900f50afe..99fe2543a 100644 --- a/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/DispatcherDaoTests.java +++ b/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/DispatcherDaoTests.java @@ -146,11 +146,12 @@ public void createHost() { RenderHost host = RenderHost.newBuilder() .setName(HOSTNAME) .setBootTime(1192369572) - .setFreeMcp(76020) + // The minimum amount of free space in the temporary directory to book a host. + .setFreeMcp(CueUtil.GB) .setFreeMem(53500) .setFreeSwap(20760) .setLoad(1) - .setTotalMcp(195430) + .setTotalMcp(CueUtil.GB4) .setTotalMem(8173264) .setTotalSwap(20960) .setNimbyEnabled(false) diff --git a/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/FrameDaoTests.java b/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/FrameDaoTests.java index 8d64d918e..6312e6502 100644 --- a/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/FrameDaoTests.java +++ b/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/FrameDaoTests.java @@ -114,11 +114,12 @@ public void create() { RenderHost host = RenderHost.newBuilder() .setName(HOST) .setBootTime(1192369572) - .setFreeMcp(76020) + // The minimum amount of free space in the temporary directory to book a host. + .setFreeMcp(CueUtil.GB) .setFreeMem(53500) .setFreeSwap(20760) .setLoad(1) - .setTotalMcp(195430) + .setTotalMcp(CueUtil.GB4) .setTotalMem(8173264) .setTotalSwap(20960) .setNimbyEnabled(false) diff --git a/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/HostDaoTests.java b/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/HostDaoTests.java index df965893b..a6261e464 100644 --- a/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/HostDaoTests.java +++ b/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/HostDaoTests.java @@ -86,12 +86,13 @@ public static RenderHost buildRenderHost(String name) { RenderHost host = RenderHost.newBuilder() .setName(name) .setBootTime(1192369572) - .setFreeMcp(7602) + // The minimum amount of free space in the temporary directory to book a host. + .setFreeMcp(CueUtil.GB) .setFreeMem(15290520) .setFreeSwap((int) CueUtil.MB512) .setLoad(1) .setNimbyEnabled(false) - .setTotalMcp(19543) + .setTotalMcp(CueUtil.GB4) .setTotalMem((int) CueUtil.GB16) .setTotalSwap((int) CueUtil.GB2) .setNimbyEnabled(false) diff --git a/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/ProcDaoTests.java b/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/ProcDaoTests.java index ab95e7e1a..f6cabc23a 100644 --- a/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/ProcDaoTests.java +++ b/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/ProcDaoTests.java @@ -118,11 +118,12 @@ public DispatchHost createHost() { RenderHost host = RenderHost.newBuilder() .setName("beta") .setBootTime(1192369572) - .setFreeMcp(76020) + // The minimum amount of free space in the temporary directory to book a host. + .setFreeMcp(CueUtil.GB) .setFreeMem(53500) .setFreeSwap(20760) .setLoad(1) - .setTotalMcp(195430) + .setTotalMcp(CueUtil.GB4) .setTotalMem((int) CueUtil.GB32) .setTotalSwap(20960) .setNimbyEnabled(false) diff --git a/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/ShowDaoTests.java b/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/ShowDaoTests.java index e44393ab6..ea0ed67b8 100644 --- a/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/ShowDaoTests.java +++ b/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/ShowDaoTests.java @@ -71,11 +71,12 @@ public DispatchHost createHost() { RenderHost host = RenderHost.newBuilder() .setName("test_host") .setBootTime(1192369572) - .setFreeMcp(76020) + // The minimum amount of free space in the temporary directory to book a host. + .setFreeMcp(CueUtil.GB) .setFreeMem(53500) .setFreeSwap(20760) .setLoad(1) - .setTotalMcp(195430) + .setTotalMcp(CueUtil.GB4) .setTotalMem((int) CueUtil.GB16) .setTotalSwap((int) CueUtil.GB16) .setNimbyEnabled(false) diff --git a/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/WhiteboardDaoTests.java b/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/WhiteboardDaoTests.java index d419b6ce9..293359f8d 100644 --- a/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/WhiteboardDaoTests.java +++ b/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/WhiteboardDaoTests.java @@ -266,11 +266,12 @@ public RenderHost getRenderHost() { RenderHost host = RenderHost.newBuilder() .setName(HOST) .setBootTime(1192369572) - .setFreeMcp(7602) + // The minimum amount of free space in the temporary directory to book a host. + .setFreeMcp(CueUtil.GB) .setFreeMem((int) Dispatcher.MEM_RESERVED_MIN * 4) .setFreeSwap(2076) .setLoad(1) - .setTotalMcp(19543) + .setTotalMcp(CueUtil.GB4) .setTotalMem((int) Dispatcher.MEM_RESERVED_MIN * 4) .setTotalSwap(2096) .setNimbyEnabled(true) diff --git a/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/CoreUnitDispatcherGpuJobTests.java b/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/CoreUnitDispatcherGpuJobTests.java index 4cc1c1f03..55bd44463 100644 --- a/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/CoreUnitDispatcherGpuJobTests.java +++ b/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/CoreUnitDispatcherGpuJobTests.java @@ -100,11 +100,12 @@ public void createHost() { RenderHost host = RenderHost.newBuilder() .setName(HOSTNAME) .setBootTime(1192369572) - .setFreeMcp(76020) + // The minimum amount of free space in the temporary directory to book a host. + .setFreeMcp(CueUtil.GB) .setFreeMem((int) CueUtil.GB8) .setFreeSwap(20760) .setLoad(1) - .setTotalMcp(195430) + .setTotalMcp(CueUtil.GB4) .setTotalMem((int) CueUtil.GB8) .setTotalSwap((int) CueUtil.GB2) .setNimbyEnabled(false) diff --git a/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/CoreUnitDispatcherGpuTests.java b/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/CoreUnitDispatcherGpuTests.java index 0a4f6b74a..c61c9553f 100644 --- a/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/CoreUnitDispatcherGpuTests.java +++ b/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/CoreUnitDispatcherGpuTests.java @@ -100,11 +100,12 @@ public void createHost() { RenderHost host = RenderHost.newBuilder() .setName(HOSTNAME) .setBootTime(1192369572) - .setFreeMcp(76020) + // The minimum amount of free space in the temporary directory to book a host. + .setFreeMcp(CueUtil.GB) .setFreeMem((int) CueUtil.GB8) .setFreeSwap(20760) .setLoad(1) - .setTotalMcp(195430) + .setTotalMcp(CueUtil.GB4) .setTotalMem((int) CueUtil.GB8) .setTotalSwap((int) CueUtil.GB2) .setNimbyEnabled(false) diff --git a/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/CoreUnitDispatcherGpusJobTests.java b/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/CoreUnitDispatcherGpusJobTests.java index 4972b8f9b..e2d1cb564 100644 --- a/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/CoreUnitDispatcherGpusJobTests.java +++ b/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/CoreUnitDispatcherGpusJobTests.java @@ -112,11 +112,12 @@ public void createHost() { RenderHost host = RenderHost.newBuilder() .setName(HOSTNAME) .setBootTime(1192369572) - .setFreeMcp(76020) + // The minimum amount of free space in the temporary directory to book a host. + .setFreeMcp(CueUtil.GB) .setFreeMem((int) CueUtil.GB8) .setFreeSwap(20760) .setLoad(0) - .setTotalMcp(195430) + .setTotalMcp(CueUtil.GB4) .setTotalMem(CueUtil.GB8) .setTotalSwap(CueUtil.GB2) .setNimbyEnabled(false) diff --git a/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/CoreUnitDispatcherTests.java b/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/CoreUnitDispatcherTests.java index adb6c404d..89112dd69 100644 --- a/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/CoreUnitDispatcherTests.java +++ b/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/CoreUnitDispatcherTests.java @@ -45,6 +45,7 @@ import com.imageworks.spcue.service.JobLauncher; import com.imageworks.spcue.service.JobManager; import com.imageworks.spcue.test.TransactionalTest; +import com.imageworks.spcue.util.CueUtil; import static org.junit.Assert.assertEquals; @@ -99,11 +100,12 @@ public void createHost() { RenderHost host = RenderHost.newBuilder() .setName(HOSTNAME) .setBootTime(1192369572) - .setFreeMcp(76020) + // The minimum amount of free space in the temporary directory to book a host. + .setFreeMcp(CueUtil.GB) .setFreeMem(53500) .setFreeSwap(20760) .setLoad(1) - .setTotalMcp(195430) + .setTotalMcp(CueUtil.GB4) .setTotalMem(8173264) .setTotalSwap(20960) .setNimbyEnabled(false) diff --git a/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/DispatchSupportTests.java b/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/DispatchSupportTests.java index 98c60fd9c..55a7806c0 100644 --- a/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/DispatchSupportTests.java +++ b/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/DispatchSupportTests.java @@ -96,11 +96,12 @@ public void createHost() { RenderHost host = RenderHost.newBuilder() .setName(HOSTNAME) .setBootTime(1192369572) - .setFreeMcp(76020) + // The minimum amount of free space in the temporary directory to book a host. + .setFreeMcp(CueUtil.GB) .setFreeMem(53500) .setFreeSwap(20760) .setLoad(0) - .setTotalMcp(195430) + .setTotalMcp(CueUtil.GB4) .setTotalMem(8173264) .setTotalSwap(20960) .setNimbyEnabled(false) diff --git a/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/FrameCompleteHandlerTests.java b/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/FrameCompleteHandlerTests.java index f022fc687..d313c5293 100644 --- a/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/FrameCompleteHandlerTests.java +++ b/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/FrameCompleteHandlerTests.java @@ -115,11 +115,12 @@ public void createHost() { RenderHost host = RenderHost.newBuilder() .setName(HOSTNAME) .setBootTime(1192369572) - .setFreeMcp(76020) + // The minimum amount of free space in the temporary directory to book a host. + .setFreeMcp(CueUtil.GB) .setFreeMem((int) CueUtil.GB8) .setFreeSwap(20760) .setLoad(0) - .setTotalMcp(195430) + .setTotalMcp(CueUtil.GB4) .setTotalMem(CueUtil.GB8) .setTotalSwap(CueUtil.GB2) .setNimbyEnabled(false) @@ -139,11 +140,12 @@ public void createHost() { RenderHost host2 = RenderHost.newBuilder() .setName(HOSTNAME2) .setBootTime(1192369572) - .setFreeMcp(76020) + // The minimum amount of free space in the temporary directory to book a host. + .setFreeMcp(CueUtil.GB) .setFreeMem((int) CueUtil.GB4) .setFreeSwap((int) CueUtil.GB4) .setLoad(0) - .setTotalMcp(195430) + .setTotalMcp(CueUtil.GB4) .setTotalMem((int) CueUtil.GB8) .setTotalSwap((int) CueUtil.GB8) .setNimbyEnabled(false) diff --git a/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/HistoryControlTests.java b/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/HistoryControlTests.java index 138a3f33c..de67ff26a 100644 --- a/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/HistoryControlTests.java +++ b/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/HistoryControlTests.java @@ -102,11 +102,12 @@ public void createHost() { RenderHost host = RenderHost.newBuilder() .setName(HOSTNAME) .setBootTime(1192369572) - .setFreeMcp(76020) + // The minimum amount of free space in the temporary directory to book a host. + .setFreeMcp(CueUtil.GB) .setFreeMem((int) CueUtil.GB8) .setFreeSwap(20760) .setLoad(0) - .setTotalMcp(195430) + .setTotalMcp(CueUtil.GB4) .setTotalMem(CueUtil.GB8) .setTotalSwap(CueUtil.GB2) .setNimbyEnabled(false) diff --git a/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/HostReportHandlerGpuTests.java b/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/HostReportHandlerGpuTests.java index dee9d0792..ce1e98ae1 100644 --- a/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/HostReportHandlerGpuTests.java +++ b/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/HostReportHandlerGpuTests.java @@ -81,11 +81,12 @@ private static RenderHost getRenderHost() { return RenderHost.newBuilder() .setName(HOSTNAME) .setBootTime(1192369572) - .setFreeMcp(76020) + // The minimum amount of free space in the temporary directory to book a host. + .setFreeMcp(CueUtil.GB) .setFreeMem(53500) .setFreeSwap(20760) .setLoad(0) - .setTotalMcp(195430) + .setTotalMcp(CueUtil.GB4) .setTotalMem(1048576L * 4096) .setTotalSwap(20960) .setNimbyEnabled(false) diff --git a/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/HostReportHandlerTests.java b/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/HostReportHandlerTests.java index d27f76c32..970b97a95 100644 --- a/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/HostReportHandlerTests.java +++ b/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/HostReportHandlerTests.java @@ -22,6 +22,7 @@ import java.io.File; import java.sql.Timestamp; import java.util.List; +import java.util.concurrent.ThreadPoolExecutor; import javax.annotation.Resource; import org.junit.Before; @@ -31,6 +32,7 @@ import org.springframework.transaction.annotation.Transactional; import com.imageworks.spcue.AllocationEntity; +import com.imageworks.spcue.CommentDetail; import com.imageworks.spcue.DispatchHost; import com.imageworks.spcue.dispatcher.Dispatcher; import com.imageworks.spcue.dispatcher.HostReportHandler; @@ -43,6 +45,7 @@ import com.imageworks.spcue.grpc.report.RenderHost; import com.imageworks.spcue.grpc.report.RunningFrameInfo; import com.imageworks.spcue.service.AdminManager; +import com.imageworks.spcue.service.CommentManager; import com.imageworks.spcue.service.HostManager; import com.imageworks.spcue.service.JobLauncher; import com.imageworks.spcue.service.JobManager; @@ -50,7 +53,10 @@ import com.imageworks.spcue.util.CueUtil; import com.imageworks.spcue.VirtualProc; +import java.util.UUID; + import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; @ContextConfiguration public class HostReportHandlerTests extends TransactionalTest { @@ -73,8 +79,16 @@ public class HostReportHandlerTests extends TransactionalTest { @Resource JobManager jobManager; + @Resource + CommentManager commentManager; + private static final String HOSTNAME = "beta"; private static final String NEW_HOSTNAME = "gamma"; + private String hostname; + private String hostname2; + private static final String SUBJECT_COMMENT_FULL_TEMP_DIR = "Host set to REPAIR for not having enough storage " + + "space on the temporary directory (mcp)"; + private static final String CUEBOT_COMMENT_USER = "cuebot"; @Before public void setTestMode() { @@ -83,7 +97,11 @@ public void setTestMode() { @Before public void createHost() { - hostManager.createHost(getRenderHost(), + hostname = UUID.randomUUID().toString().substring(0, 8); + hostname2 = UUID.randomUUID().toString().substring(0, 8); + hostManager.createHost(getRenderHost(hostname, HardwareState.UP), + adminManager.findAllocationDetail("spi","general")); + hostManager.createHost(getRenderHost(hostname2, HardwareState.UP), adminManager.findAllocationDetail("spi","general")); } @@ -96,26 +114,51 @@ private static CoreDetail getCoreDetail(int total, int idle, int booked, int loc .build(); } - private DispatchHost getHost() { - return hostManager.findDispatchHost(HOSTNAME); + private DispatchHost getHost(String hostname) { + return hostManager.findDispatchHost(hostname); } - private static RenderHost getRenderHost() { + private static RenderHost getRenderHost(String hostname, HardwareState state) { return RenderHost.newBuilder() - .setName(HOSTNAME) + .setName(hostname) .setBootTime(1192369572) - .setFreeMcp(76020) + // The minimum amount of free space in the temporary directory to book a host. + .setFreeMcp(CueUtil.GB) .setFreeMem((int) CueUtil.GB8) .setFreeSwap(20760) .setLoad(0) - .setTotalMcp(195430) + .setTotalMcp(CueUtil.GB4) .setTotalMem(CueUtil.GB8) .setTotalSwap(CueUtil.GB2) .setNimbyEnabled(false) .setNumProcs(2) .setCoresPerProc(100) .addTags("test") - .setState(HardwareState.UP) + .setState(state) + .setFacility("spi") + .putAttributes("SP_OS", "Linux") + .setFreeGpuMem((int) CueUtil.MB512) + .setTotalGpuMem((int) CueUtil.MB512) + .build(); + } + + private static RenderHost getRenderHost(String hostname, HardwareState state, Long freeTempDir) { + return RenderHost.newBuilder() + .setName(hostname) + .setBootTime(1192369572) + // The minimum amount of free space in the temporary directory to book a host. + .setFreeMcp(freeTempDir) + .setFreeMem((int) CueUtil.GB8) + .setFreeSwap(20760) + .setLoad(0) + .setTotalMcp(freeTempDir * 4) + .setTotalMem(CueUtil.GB8) + .setTotalSwap(CueUtil.GB2) + .setNimbyEnabled(false) + .setNumProcs(2) + .setCoresPerProc(100) + .addTags("test") + .setState(state) .setFacility("spi") .putAttributes("SP_OS", "Linux") .setFreeGpuMem((int) CueUtil.MB512) @@ -127,11 +170,12 @@ private static RenderHost getNewRenderHost(String tags) { return RenderHost.newBuilder() .setName(NEW_HOSTNAME) .setBootTime(1192369572) - .setFreeMcp(76020) + // The minimum amount of free space in the temporary directory to book a host. + .setFreeMcp(CueUtil.GB) .setFreeMem(53500) .setFreeSwap(20760) .setLoad(0) - .setTotalMcp(195430) + .setTotalMcp(CueUtil.GB4) .setTotalMem(8173264) .setTotalSwap(20960) .setNimbyEnabled(false) @@ -149,17 +193,40 @@ private static RenderHost getNewRenderHost(String tags) { @Test @Transactional @Rollback(true) - public void testHandleHostReport() { - boolean isBoot = false; + public void testHandleHostReport() throws InterruptedException { CoreDetail cores = getCoreDetail(200, 200, 0, 0); - HostReport report = HostReport.newBuilder() - .setHost(getRenderHost()) + HostReport report1 = HostReport.newBuilder() + .setHost(getRenderHost(hostname, HardwareState.UP)) .setCoreInfo(cores) .build(); + HostReport report2 = HostReport.newBuilder() + .setHost(getRenderHost(hostname2, HardwareState.UP)) + .setCoreInfo(cores) + .build(); + HostReport report1_2 = HostReport.newBuilder() + .setHost(getRenderHost(hostname, HardwareState.UP)) + .setCoreInfo(getCoreDetail(200, 200, 100, 0)) + .build(); - hostReportHandler.handleHostReport(report, isBoot); - DispatchHost host = getHost(); - assertEquals(host.lockState, LockState.OPEN); + hostReportHandler.handleHostReport(report1, false); + DispatchHost host = getHost(hostname); + assertEquals(LockState.OPEN, host.lockState); + assertEquals(HardwareState.UP, host.hardwareState); + hostReportHandler.handleHostReport(report1_2, false); + host = getHost(hostname); + assertEquals(HardwareState.UP, host.hardwareState); + + // Test Queue thread handling + ThreadPoolExecutor queue = hostReportHandler.getReportQueue(); + // Make sure jobs flow normally without any nullpointer exception + // Expecting results from a ThreadPool based class on JUnit is tricky + // A future test will be developed in the future to better address the behavior of + // this feature + hostReportHandler.queueHostReport(report1); // HOSTNAME + hostReportHandler.queueHostReport(report2); // HOSTNAME2 + hostReportHandler.queueHostReport(report1); // HOSTNAME + hostReportHandler.queueHostReport(report1); // HOSTNAME + hostReportHandler.queueHostReport(report1_2); // HOSTNAME } @Test @@ -228,6 +295,138 @@ public void testHandleHostReportWithNonExistentTags() { assertEquals(host.getAllocationId(), alloc.id); } + @Test + @Transactional + @Rollback(true) + public void testHandleHostReportWithFullTemporaryDirectories() { + // Create CoreDetail + CoreDetail cores = getCoreDetail(200, 200, 0, 0); + + /* + * Test 1: + * Precondition: + * - HardwareState=UP + * Action: + * - Receives a HostReport with freeTempDir < dispatcher.min_bookable_free_temp_dir_kb (opencue.properties) + * Postcondition: + * - Host hardwareState changes to REPAIR + * - A comment is created with subject=SUBJECT_COMMENT_FULL_TEMP_DIR and user=CUEBOT_COMMENT_USER + * */ + // Create HostReport + HostReport report1 = HostReport.newBuilder() + .setHost(getRenderHost(hostname, HardwareState.UP, 1024L)) + .setCoreInfo(cores) + .build(); + // Call handleHostReport() => Create the comment with subject=SUBJECT_COMMENT_FULL_TEMP_DIR and change the + // host's hardwareState to REPAIR + hostReportHandler.handleHostReport(report1, false); + // Get host + DispatchHost host = getHost(hostname); + // Get list of comments by host, user, and subject + List comments = commentManager.getCommentsByHostUserAndSubject(host, CUEBOT_COMMENT_USER, + SUBJECT_COMMENT_FULL_TEMP_DIR); + // Check if there is 1 comment + assertEquals(comments.size(), 1); + // Get host comment + CommentDetail comment = comments.get(0); + // Check if the comment has the user = CUEBOT_COMMENT_USER + assertEquals(comment.user, CUEBOT_COMMENT_USER); + // Check if the comment has the subject = SUBJECT_COMMENT_FULL_TEMP_DIR + assertEquals(comment.subject, SUBJECT_COMMENT_FULL_TEMP_DIR); + // Check host lock state + assertEquals(LockState.OPEN, host.lockState); + // Check if host hardware state is REPAIR + assertEquals(HardwareState.REPAIR, host.hardwareState); + // Test Queue thread handling + ThreadPoolExecutor queue = hostReportHandler.getReportQueue(); + // Make sure jobs flow normally without any nullpointer exception + hostReportHandler.queueHostReport(report1); // HOSTNAME + hostReportHandler.queueHostReport(report1); // HOSTNAME + + /* + * Test 2: + * Precondition: + * - HardwareState=REPAIR + * - There is a comment for the host with subject=SUBJECT_COMMENT_FULL_TEMP_DIR and user=CUEBOT_COMMENT_USER + * Action: + * - Receives a HostReport with freeTempDir >= dispatcher.min_bookable_free_temp_dir_kb (opencue.properties) + * Postcondition: + * - Host hardwareState changes to UP + * - Comment with subject=SUBJECT_COMMENT_FULL_TEMP_DIR and user=CUEBOT_COMMENT_USER gets deleted + * */ + // Set the host freeTempDir to the minimum size required = 1GB (1048576 KB) + HostReport report2 = HostReport.newBuilder() + .setHost(getRenderHost(hostname, HardwareState.UP, 1048576L)) + .setCoreInfo(cores) + .build(); + // Call handleHostReport() => Delete the comment with subject=SUBJECT_COMMENT_FULL_TEMP_DIR and change the + // host's hardwareState to UP + hostReportHandler.handleHostReport(report2, false); + // Get host + host = getHost(hostname); + // Get list of comments by host, user, and subject + comments = commentManager.getCommentsByHostUserAndSubject(host, CUEBOT_COMMENT_USER, + SUBJECT_COMMENT_FULL_TEMP_DIR); + // Check if there is no comment associated with the host + assertEquals(comments.size(), 0); + // Check host lock state + assertEquals(LockState.OPEN, host.lockState); + // Check if host hardware state is UP + assertEquals(HardwareState.UP, host.hardwareState); + // Test Queue thread handling + queue = hostReportHandler.getReportQueue(); + // Make sure jobs flow normally without any nullpointer exception + hostReportHandler.queueHostReport(report1); // HOSTNAME + hostReportHandler.queueHostReport(report1); // HOSTNAME + } + + @Test + @Transactional + @Rollback(true) + public void testHandleHostReportWithHardwareStateRepairNotRelatedToFullTempDir() { + // Create CoreDetail + CoreDetail cores = getCoreDetail(200, 200, 0, 0); + + /* + * Test if host.hardwareState == HardwareState.REPAIR + * (Not related to freeMcp < dispatcher.min_bookable_free_mcp_kb (opencue.properties)) + * + * - There is no comment with subject=SUBJECT_COMMENT_FULL_MCP_DIR and user=CUEBOT_COMMENT_USER associated with + * the host + * The host.hardwareState continue as HardwareState.REPAIR + * */ + // Create HostReport + HostReport report = HostReport.newBuilder() + .setHost(getRenderHost(hostname, HardwareState.UP, 1048576L)) + .setCoreInfo(cores) + .build(); + // Get host + DispatchHost host = getHost(hostname); + // Host's HardwareState set to REPAIR + hostManager.setHostState(host, HardwareState.REPAIR); + host.hardwareState = HardwareState.REPAIR; + // Get list of comments by host, user, and subject + List hostComments = commentManager.getCommentsByHostUserAndSubject(host, CUEBOT_COMMENT_USER, + SUBJECT_COMMENT_FULL_TEMP_DIR); + // Check if there is no comment + assertEquals(hostComments.size(), 0); + // There is no comment to delete + boolean commentsDeleted = commentManager.deleteCommentByHostUserAndSubject(host, + CUEBOT_COMMENT_USER, SUBJECT_COMMENT_FULL_TEMP_DIR); + assertFalse(commentsDeleted); + // Call handleHostReport() + hostReportHandler.handleHostReport(report, false); + // Check host lock state + assertEquals(LockState.OPEN, host.lockState); + // Check if host hardware state is REPAIR + assertEquals(HardwareState.REPAIR, host.hardwareState); + // Test Queue thread handling + ThreadPoolExecutor queueThread = hostReportHandler.getReportQueue(); + // Make sure jobs flow normally without any nullpointer exception + hostReportHandler.queueHostReport(report); // HOSTNAME + hostReportHandler.queueHostReport(report); // HOSTNAME + } + @Test @Transactional @Rollback(true) @@ -235,7 +434,7 @@ public void testMemoryAndLlu() { jobLauncher.testMode = true; jobLauncher.launch(new File("src/test/resources/conf/jobspec/jobspec_simple.xml")); - DispatchHost host = getHost(); + DispatchHost host = getHost(hostname); List procs = dispatcher.dispatchHost(host); assertEquals(1, procs.size()); VirtualProc proc = procs.get(0); @@ -252,7 +451,7 @@ public void testMemoryAndLlu() { .setMaxRss(420000) .build(); HostReport report = HostReport.newBuilder() - .setHost(getRenderHost()) + .setHost(getRenderHost(hostname, HardwareState.UP)) .setCoreInfo(cores) .addFrames(info) .build(); diff --git a/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/LocalDispatcherTests.java b/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/LocalDispatcherTests.java index 97a270085..a7218b47a 100644 --- a/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/LocalDispatcherTests.java +++ b/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/LocalDispatcherTests.java @@ -95,11 +95,12 @@ public void createHost() { RenderHost host = RenderHost.newBuilder() .setName(HOSTNAME) .setBootTime(1192369572) - .setFreeMcp(76020) + // The minimum amount of free space in the temporary directory to book a host. + .setFreeMcp(CueUtil.GB) .setFreeMem(53500) .setFreeSwap(20760) .setLoad(0) - .setTotalMcp(195430) + .setTotalMcp(CueUtil.GB4) .setTotalMem(8173264) .setTotalSwap(20960) .setNimbyEnabled(false) diff --git a/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/RedirectManagerTests.java b/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/RedirectManagerTests.java index 25ccf69c5..70e3db4af 100644 --- a/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/RedirectManagerTests.java +++ b/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/RedirectManagerTests.java @@ -60,6 +60,7 @@ import com.imageworks.spcue.service.RedirectService; import com.imageworks.spcue.service.Whiteboard; import com.imageworks.spcue.util.Convert; +import com.imageworks.spcue.util.CueUtil; import static org.hamcrest.Matchers.emptyString; import static org.hamcrest.Matchers.is; @@ -137,11 +138,12 @@ public void createHost() { RenderHost host = RenderHost.newBuilder() .setName(HOSTNAME) .setBootTime(1192369572) - .setFreeMcp(76020) + // The minimum amount of free space in the temporary directory to book a host. + .setFreeMcp(CueUtil.GB) .setFreeMem(53500) .setFreeSwap(20760) .setLoad(1) - .setTotalMcp(195430) + .setTotalMcp(CueUtil.GB4) .setTotalMem(8173264) .setTotalSwap(20960) .setNimbyEnabled(false) diff --git a/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/StrandedCoreTests.java b/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/StrandedCoreTests.java index 4211c9866..7d02d44e8 100644 --- a/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/StrandedCoreTests.java +++ b/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/StrandedCoreTests.java @@ -43,6 +43,7 @@ import com.imageworks.spcue.service.JobLauncher; import com.imageworks.spcue.service.JobManager; import com.imageworks.spcue.test.TransactionalTest; +import com.imageworks.spcue.util.CueUtil; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; @@ -98,11 +99,12 @@ public void createHost() { RenderHost host = RenderHost.newBuilder() .setName(HOSTNAME) .setBootTime(1192369572) - .setFreeMcp(76020) + // The minimum amount of free space in the temporary directory to book a host. + .setFreeMcp(CueUtil.GB) .setFreeMem(53500) .setFreeSwap(20760) .setLoad(1) - .setTotalMcp(195430) + .setTotalMcp(CueUtil.GB4) .setTotalMem(8173264) .setTotalSwap(20960) .setNimbyEnabled(false) diff --git a/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/TestBookingQueue.java b/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/TestBookingQueue.java index 74b21c102..7654570a0 100644 --- a/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/TestBookingQueue.java +++ b/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/TestBookingQueue.java @@ -64,11 +64,12 @@ public void create() { RenderHost host = RenderHost.newBuilder() .setName(HOSTNAME) .setBootTime(1192369572) - .setFreeMcp(76020) + // The minimum amount of free space in the temporary directory to book a host. + .setFreeMcp(CueUtil.GB) .setFreeMem(53500) .setFreeSwap(20760) .setLoad(1) - .setTotalMcp(195430) + .setTotalMcp(CueUtil.GB4) .setTotalMem(8173264) .setTotalSwap(20960) .setNimbyEnabled(false) diff --git a/cuebot/src/test/java/com/imageworks/spcue/test/service/BookingManagerTests.java b/cuebot/src/test/java/com/imageworks/spcue/test/service/BookingManagerTests.java index 9b6813c33..1e894eb1c 100644 --- a/cuebot/src/test/java/com/imageworks/spcue/test/service/BookingManagerTests.java +++ b/cuebot/src/test/java/com/imageworks/spcue/test/service/BookingManagerTests.java @@ -112,11 +112,12 @@ public DispatchHost createHost() { RenderHost host = RenderHost.newBuilder() .setName("test_host") .setBootTime(1192369572) - .setFreeMcp(76020) + // The minimum amount of free space in the temporary directory to book a host. + .setFreeMcp(CueUtil.GB) .setFreeMem(53500) .setFreeSwap(20760) .setLoad(1) - .setTotalMcp(195430) + .setTotalMcp(CueUtil.GB4) .setTotalMem((int) CueUtil.GB16) .setTotalSwap((int) CueUtil.GB16) .setNimbyEnabled(false) diff --git a/cuebot/src/test/java/com/imageworks/spcue/test/service/HostManagerTests.java b/cuebot/src/test/java/com/imageworks/spcue/test/service/HostManagerTests.java index cf86e5362..29970441d 100644 --- a/cuebot/src/test/java/com/imageworks/spcue/test/service/HostManagerTests.java +++ b/cuebot/src/test/java/com/imageworks/spcue/test/service/HostManagerTests.java @@ -101,11 +101,12 @@ public DispatchHost createHost() { RenderHost host = RenderHost.newBuilder() .setName(HOST_NAME) .setBootTime(1192369572) - .setFreeMcp(7602) + // The minimum amount of free space in the temporary directory to book a host. + .setFreeMcp(CueUtil.GB) .setFreeMem(15290520) .setFreeSwap(2076) .setLoad(1) - .setTotalMcp(19543) + .setTotalMcp(CueUtil.GB4) .setTotalMem((int) CueUtil.GB16) .setTotalSwap(2076) .setNimbyEnabled(true) diff --git a/cuebot/src/test/java/com/imageworks/spcue/test/service/JobManagerTests.java b/cuebot/src/test/java/com/imageworks/spcue/test/service/JobManagerTests.java index 3be56bf06..2ea9b5dde 100644 --- a/cuebot/src/test/java/com/imageworks/spcue/test/service/JobManagerTests.java +++ b/cuebot/src/test/java/com/imageworks/spcue/test/service/JobManagerTests.java @@ -129,11 +129,12 @@ public DispatchHost createHost() { RenderHost host = RenderHost.newBuilder() .setName("test_host") .setBootTime(1192369572) - .setFreeMcp(76020) + // The minimum amount of free space in the temporary directory to book a host. + .setFreeMcp(CueUtil.GB) .setFreeMem(53500) .setFreeSwap(20760) .setLoad(1) - .setTotalMcp(195430) + .setTotalMcp(CueUtil.GB4) .setTotalMem((int) CueUtil.GB16) .setTotalSwap((int) CueUtil.GB16) .setNimbyEnabled(false) diff --git a/cuebot/src/test/java/com/imageworks/spcue/test/service/OwnerManagerTests.java b/cuebot/src/test/java/com/imageworks/spcue/test/service/OwnerManagerTests.java index 224dcac75..51dcafec4 100644 --- a/cuebot/src/test/java/com/imageworks/spcue/test/service/OwnerManagerTests.java +++ b/cuebot/src/test/java/com/imageworks/spcue/test/service/OwnerManagerTests.java @@ -69,11 +69,12 @@ public DispatchHost createHost() { RenderHost host = RenderHost.newBuilder() .setName("test_host") .setBootTime(1192369572) - .setFreeMcp(76020) + // The minimum amount of free space in the temporary directory to book a host. + .setFreeMcp(CueUtil.GB) .setFreeMem(53500) .setFreeSwap(20760) .setLoad(1) - .setTotalMcp(195430) + .setTotalMcp(CueUtil.GB4) .setTotalMem((int) CueUtil.GB16) .setTotalSwap((int) CueUtil.GB16) .setNimbyEnabled(true) diff --git a/cuebot/src/test/resources/opencue.properties b/cuebot/src/test/resources/opencue.properties index 334408470..00d0c4463 100644 --- a/cuebot/src/test/resources/opencue.properties +++ b/cuebot/src/test/resources/opencue.properties @@ -64,3 +64,9 @@ dispatcher.kill_queue.queue_capacity=1000 dispatcher.booking_queue.core_pool_size=6 dispatcher.booking_queue.max_pool_size=6 dispatcher.booking_queue.queue_capacity=1000 + +# The minimum amount of free space in the temporary directory (mcp) to book a host. +# E.g: 1G = 1048576 kB => dispatcher.min_bookable_free_temp_dir_kb=1048576 +# Default = 1G = 1048576 kB +# If equals to -1, it means the feature is turned off +dispatcher.min_bookable_free_temp_dir_kb=1048576 \ No newline at end of file From a94393aa3d408b38894403ce461ca349da41093d Mon Sep 17 00:00:00 2001 From: Brian Cipriano Date: Wed, 27 Sep 2023 18:39:15 -0400 Subject: [PATCH 38/56] TSC meeting notes. (#1318) --- tsc/meetings/2023-07-19.md | 45 ++++++++++++++++++++++++++++++++++++++ tsc/meetings/2023-09-27.md | 24 ++++++++++++++++++++ 2 files changed, 69 insertions(+) create mode 100644 tsc/meetings/2023-07-19.md create mode 100644 tsc/meetings/2023-09-27.md diff --git a/tsc/meetings/2023-07-19.md b/tsc/meetings/2023-07-19.md new file mode 100644 index 000000000..61051f9c9 --- /dev/null +++ b/tsc/meetings/2023-07-19.md @@ -0,0 +1,45 @@ +# OpenCue TSC Meeting Notes 19 July 2023 + +Secretary: Brian Cipriano + +Agenda/Notes: + +* CI pipeline updates + * https://github.com/AcademySoftwareFoundation/OpenCue/pull/1305 + * Added new VFX ref platform years, retired old ones. + * Added CY2023. + * Keet CY2022. + * Drop CY2021, CY2020. + * Keet CY2019, but repurposed it as an explicitly-named Python 2 test. + * Disabled GUI tests on older platforms due to flakes, we'll keep running them on CY2023. + * A few other minor dependency upgrades and fixes. +* New release v0.22.14. + * We needed to get the latest database query fixes into an official release, newer versions of + Postgres that trigger those issues are more common now. + * Release includes: + * PySide6 in CueGUI, still needed for CueSubmit. + * Config / env var cleanup. Published a new doc page covering + this: https://www.opencue.io/docs/other-guides/configuring-opencue/ +* Enable GPU booking + * https://lists.aswf.io/g/opencue-user/topic/local_deployment_issues/100008713 + * Any ideas for this user? + * You need to have the nvidia-smi tool on RQD to detect GPU hardware. + * Once we figure this out, we should write up a doc page on how to enable GPU. + * If the user is on Docker, they may need to use the nvidia base image. +* Minimum bookable free mcp + * https://github.com/AcademySoftwareFoundation/OpenCue/pull/1306 + * Enforce minimum mcp (scratch space) for all hosts, take host offline if low on space. + * Brian to review. + * Ideally we should avoid spreading the "mcp" terminology, but this is a much larger project, + let's just avoid it where we can. +* Siggraph + * Nothing official planned, some folks attending virtually. +* SPI updates + * Finally up-to-date with “current” version of github. + * Performance issues on DispatchQuery. + * Using database migrations starting at v1000, this works because migrations are all applied + manually anyway, not via e.g. Flyway. + * When we create migrations, if you rename a field, you need to copy the value as well. +* Blender plugin update + * Added the ability to refresh/update opencue code from the addon. + * Brian to follow up on email thread. diff --git a/tsc/meetings/2023-09-27.md b/tsc/meetings/2023-09-27.md new file mode 100644 index 000000000..562f31f28 --- /dev/null +++ b/tsc/meetings/2023-09-27.md @@ -0,0 +1,24 @@ +# OpenCue TSC Meeting Notes 27 Sep 2023 + +Secretary: Brian Cipriano + +Agenda/Notes: + +* OOM protection logic: https://github.com/AcademySoftwareFoundation/OpenCue/pull/1317 + * Reworks OOM protection using percentages instead of hardcoded values, helps for larger hosts. + * Draft status for now, working on some more testing. + * Cuebot only for now. Solves 99% of cases but 1% still have a race condition because RQD does + not send an exit code indicating OOM. RQD fixes coming next. +* Reserve all cores / negative cores: https://github.com/AcademySoftwareFoundation/OpenCue/pull/1313 + * Diego / SPI to discuss. + * Definitely needs to be wrapped in a flag. +* Nimby override: https://github.com/AcademySoftwareFoundation/OpenCue/pull/1311 + * Let's double check the fallback behavior is working and there's not some other error. + * PR could be a good idea anyway, we know pynput isn't installed in certain environments and the + warning in the logs can be confusing. +* SPI update + * Merging CueGUI updates. Config file change and qtpy change. +* Blender plugin + * Running in a container working now. + * Docker run flag to mount the host network worked. Let's update the linux contributing page. + * Job submitted but didn't show up in the job list. RQD can't find blender command, debugging. From 226a27dbd9c3bdbc9effdddd03a4904bfa02824d Mon Sep 17 00:00:00 2001 From: Ramon Figueiredo Date: Wed, 27 Sep 2023 16:36:49 -0700 Subject: [PATCH 39/56] [cuegui] Bug fix missing jobs on MonitorJobs (#1312) * [cuegui] Bug fix missing jobs on MonitorJobs When the option group dependent is checked, jobs without dependency are being omitted. * [cuegui] Bug fix missing jobs on MonitorJobs - Remove dependent if it has the same name as the job. This avoids missing jobs on MonitorJobs. Remove the parent job is necessary to avoid remove the parent job and all the dependents when del self.__load[j] is called - Use list comprehension to remove dependent if it has the same name as the job - List comprehension avoid many nested blocks and follow the project code style * [cuegui] Bug fix when unmonitor finished jobs with dependents - AbstractTreeWidget.py > _removeItem(): Check if object ID exists before delete it * Merge master into fix_missing_jobs_on_monitor_jobs * [cuegui] Bug fix missing jobs on MonitorJobs - When the option group dependent is checked, jobs without dependency are omitted. - Code refactoring --- cuegui/cuegui/JobMonitorTree.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/cuegui/cuegui/JobMonitorTree.py b/cuegui/cuegui/JobMonitorTree.py index d92c44afc..7b829e959 100644 --- a/cuegui/cuegui/JobMonitorTree.py +++ b/cuegui/cuegui/JobMonitorTree.py @@ -285,6 +285,14 @@ def addJob(self, job, timestamp=None, loading_from_config=False): dep = self.__menuActions.jobs( ).getRecursiveDependentJobs([newJobObj], active_only=active_only) + + # Remove dependent if it has the same name as the job + # - This avoids missing jobs on MonitorJobs + # - Remove the parent job is necessary to avoid remove + # the parent job and all the dependents + # in the step 2 below + dep = [j for j in dep if j.data.name != newJobObj.data.name] + self.__dependentJobs[jobKey] = dep # we'll also store a reversed dictionary for # dependencies with the dependent as key and the main From 287dc1ab53607f50b79ef70cb9b1f2121ce588f2 Mon Sep 17 00:00:00 2001 From: Nuwan Jayawardene Date: Tue, 3 Oct 2023 20:44:25 +0530 Subject: [PATCH 40/56] [rqd] Disable pynput in the Docker image to prevent confusing warning logs. (#1311) --- rqd/Dockerfile | 4 ++++ rqd/rqd/rqconstants.py | 2 ++ 2 files changed, 6 insertions(+) diff --git a/rqd/Dockerfile b/rqd/Dockerfile index 15f5ac21c..93b222aa7 100644 --- a/rqd/Dockerfile +++ b/rqd/Dockerfile @@ -52,6 +52,10 @@ RUN versioned_name="rqd-$(cat ./VERSION)-all" \ && tar -cvzf $versioned_name.tar.gz $versioned_name/* \ && ln -s $versioned_name rqd +RUN mkdir -p /etc/opencue +RUN echo "[Override]" > /etc/opencue/rqd.conf +RUN echo "USE_NIMBY_PYNPUT=false" >> /etc/opencue/rqd.conf + # RQD gRPC server EXPOSE 8444 diff --git a/rqd/rqd/rqconstants.py b/rqd/rqd/rqconstants.py index fe793970e..1daf2f993 100644 --- a/rqd/rqd/rqconstants.py +++ b/rqd/rqd/rqconstants.py @@ -172,6 +172,8 @@ CUEBOT_HOSTNAME = config.get(__section, "OVERRIDE_CUEBOT") if config.has_option(__section, "OVERRIDE_NIMBY"): OVERRIDE_NIMBY = config.getboolean(__section, "OVERRIDE_NIMBY") + if config.has_option(__section, "USE_NIMBY_PYNPUT"): + USE_NIMBY_PYNPUT = config.getboolean(__section, "USE_NIMBY_PYNPUT") if config.has_option(__section, "OVERRIDE_HOSTNAME"): OVERRIDE_HOSTNAME = config.get(__section, "OVERRIDE_HOSTNAME") if config.has_option(__section, "GPU"): From b203380ba4361e887692ae89f9c5a371e4b7177c Mon Sep 17 00:00:00 2001 From: Nuwan Jayawardene Date: Tue, 3 Oct 2023 20:46:07 +0530 Subject: [PATCH 41/56] [rqd] Add Blender to PATH and set RQD_USE_PATH_ENV_VAR in sample Docker image. (#1319) --- samples/rqd/blender/Dockerfile | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/samples/rqd/blender/Dockerfile b/samples/rqd/blender/Dockerfile index 208f3c7a7..bb2235613 100644 --- a/samples/rqd/blender/Dockerfile +++ b/samples/rqd/blender/Dockerfile @@ -30,5 +30,11 @@ RUN tar -xvf blender.tar.xz \ RUN rm blender.tar.xz +# Add Blender path as environment variable +ENV PATH=$PATH:${BLENDER_INSTALL_DIR} + +# Allows RQD to read Blender install directory in PATH env. variable +RUN echo "RQD_USE_PATH_ENV_VAR=true" >> /etc/opencue/rqd.conf + # Verify Blender installation -RUN ${BLENDER_INSTALL_DIR}/blender --version +RUN blender --version From e3136f4a0b571698043c807b85c2a909b84a1c58 Mon Sep 17 00:00:00 2001 From: Diego Tavares da Silva Date: Wed, 8 Nov 2023 09:05:14 -0800 Subject: [PATCH 42/56] Oom protection (#1321) * The current logic relies on hardcoded values which are not suitable for large hosts. The new logic takes into account the size of hosts and also tries to be more aggressive with misbehaving frames. Prevent host from entering an OOM state where oom-killer might start killing important OS processes. The kill logic will kick in one of the following conditions is met: Host has less than OOM_MEMORY_LEFT_THRESHOLD_PERCENT memory available A frame is taking more than OOM_FRAME_OVERBOARD_PERCENT of what it had reserved For frames that are using more than they had reserved but not above the threshold, negotiate expanding the reservations with other frames on the same host (cherry picked from commit e88a5295f23bd927614de6d5af6a09d496d3e6ac) * Frames killed for OOM should be retried (cherry picked from commit b88f7bcb1ad43f83fb8357576c33483dc2bf4952) * OOM_FRAME_OVERBOARD_ALLOWED_THRESHOLD can be deactivated with -1 (cherry picked from commit 647e75e2254c7a7ff68c544e438080f412bf04c1) * Limit the number of kill retries There's an error condition on rqd where a frame that cannot be killed will end up preventing the host from picking up new jobs. This logic limits the number of repeated killRequests to give host a chance to pick up new jobs. At the same time, blocked frames are logged to spcue.log to be handled manually. (cherry picked from commit aea4864ef66aca494fb455a7c103e4a832b63d41) * Fix merge conflicts * Handle MR comments * Minor improvements to the logic Signed-off-by: Diego Tavares --------- Signed-off-by: Diego Tavares --- .../com/imageworks/spcue/FrameInterface.java | 1 - .../com/imageworks/spcue/dao/FrameDao.java | 7 + .../com/imageworks/spcue/dao/HostDao.java | 9 - .../spcue/dao/postgres/FrameDaoJdbc.java | 18 + .../spcue/dao/postgres/HostDaoJdbc.java | 9 - .../spcue/dao/postgres/ProcDaoJdbc.java | 2 +- .../spcue/dispatcher/DispatchSupport.java | 8 + .../dispatcher/DispatchSupportService.java | 13 +- .../spcue/dispatcher/Dispatcher.java | 10 +- .../dispatcher/FrameCompleteHandler.java | 73 +-- .../spcue/dispatcher/HostReportHandler.java | 519 +++++++++++------- .../commands/DispatchRqdKillFrame.java | 20 +- .../commands/DispatchRqdKillFrameMemory.java | 78 +++ .../imageworks/spcue/service/HostManager.java | 9 - .../spcue/service/HostManagerService.java | 6 - .../spring/applicationContext-service.xml | 1 - cuebot/src/main/resources/opencue.properties | 16 +- .../spcue/test/dao/postgres/HostDaoTests.java | 18 - .../dispatcher/FrameCompleteHandlerTests.java | 6 +- .../dispatcher/HostReportHandlerGpuTests.java | 10 +- .../dispatcher/HostReportHandlerTests.java | 243 ++++++-- .../conf/jobspec/jobspec_multiple_frames.xml | 48 ++ cuebot/src/test/resources/opencue.properties | 13 +- rqd/rqd/rqdservicers.py | 2 + rqd/rqd/rqnetwork.py | 1 + 25 files changed, 764 insertions(+), 376 deletions(-) create mode 100644 cuebot/src/main/java/com/imageworks/spcue/dispatcher/commands/DispatchRqdKillFrameMemory.java create mode 100644 cuebot/src/test/resources/conf/jobspec/jobspec_multiple_frames.xml diff --git a/cuebot/src/main/java/com/imageworks/spcue/FrameInterface.java b/cuebot/src/main/java/com/imageworks/spcue/FrameInterface.java index eff22768f..945685444 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/FrameInterface.java +++ b/cuebot/src/main/java/com/imageworks/spcue/FrameInterface.java @@ -1,4 +1,3 @@ - /* * Copyright Contributors to the OpenCue Project * diff --git a/cuebot/src/main/java/com/imageworks/spcue/dao/FrameDao.java b/cuebot/src/main/java/com/imageworks/spcue/dao/FrameDao.java index 7c2e3c050..4dbb0e987 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dao/FrameDao.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dao/FrameDao.java @@ -202,6 +202,13 @@ boolean updateFrameStopped(FrameInterface frame, FrameState state, int exitStatu * @return */ boolean updateFrameCleared(FrameInterface frame); + /** + * Sets a frame exitStatus to EXIT_STATUS_MEMORY_FAILURE + * + * @param frame + * @return whether the frame has been updated + */ + boolean updateFrameMemoryError(FrameInterface frame); /** * Sets a frame to an unreserved waiting state. diff --git a/cuebot/src/main/java/com/imageworks/spcue/dao/HostDao.java b/cuebot/src/main/java/com/imageworks/spcue/dao/HostDao.java index 5ed18947e..94ba316b1 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dao/HostDao.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dao/HostDao.java @@ -252,15 +252,6 @@ public interface HostDao { */ void updateThreadMode(HostInterface host, ThreadMode mode); - /** - * When a host is in kill mode that means its 256MB+ into the swap and the - * the worst memory offender is killed. - * - * @param h HostInterface - * @return boolean - */ - boolean isKillMode(HostInterface h); - /** * Update the specified host's hardware information. * diff --git a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/FrameDaoJdbc.java b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/FrameDaoJdbc.java index 3c752ad96..21c197a3b 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/FrameDaoJdbc.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/FrameDaoJdbc.java @@ -155,6 +155,24 @@ public boolean updateFrameCleared(FrameInterface frame) { return updateFrame(frame, Dispatcher.EXIT_STATUS_FRAME_CLEARED) > 0; } + private static final String UPDATE_FRAME_MEMORY_ERROR = + "UPDATE "+ + "frame "+ + "SET " + + "int_exit_status = ?, " + + "int_version = int_version + 1 " + + "WHERE " + + "frame.pk_frame = ? "; + @Override + public boolean updateFrameMemoryError(FrameInterface frame) { + int result = getJdbcTemplate().update( + UPDATE_FRAME_MEMORY_ERROR, + Dispatcher.EXIT_STATUS_MEMORY_FAILURE, + frame.getFrameId()); + + return result > 0; + } + private static final String UPDATE_FRAME_STARTED = "UPDATE " + "frame " + diff --git a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/HostDaoJdbc.java b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/HostDaoJdbc.java index 6fe898b44..f703416b2 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/HostDaoJdbc.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/HostDaoJdbc.java @@ -612,15 +612,6 @@ public void updateHostOs(HostInterface host, String os) { os, host.getHostId()); } - @Override - public boolean isKillMode(HostInterface h) { - return getJdbcTemplate().queryForObject( - "SELECT COUNT(1) FROM host_stat WHERE pk_host = ? " + - "AND int_swap_total - int_swap_free > ? AND int_mem_free < ?", - Integer.class, h.getHostId(), Dispatcher.KILL_MODE_SWAP_THRESHOLD, - Dispatcher.KILL_MODE_MEM_THRESHOLD) > 0; - } - @Override public int getStrandedCoreUnits(HostInterface h) { try { diff --git a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/ProcDaoJdbc.java b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/ProcDaoJdbc.java index 5af292fb3..8f6322690 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/ProcDaoJdbc.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/ProcDaoJdbc.java @@ -564,7 +564,7 @@ public boolean increaseReservedMemory(ProcInterface p, long value) { value, p.getProcId(), value) == 1; } catch (Exception e) { // check by trigger erify_host_resources - throw new ResourceReservationFailureException("failed to increase memory reserveration for proc " + throw new ResourceReservationFailureException("failed to increase memory reservation for proc " + p.getProcId() + " to " + value + ", proc does not have that much memory to spare."); } } diff --git a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/DispatchSupport.java b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/DispatchSupport.java index 4accd0f8d..aa20e6266 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/DispatchSupport.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/DispatchSupport.java @@ -415,6 +415,14 @@ List findNextDispatchFrames(LayerInterface layer, VirtualProc pro */ void clearFrame(DispatchFrame frame); + /** + * Sets the frame state exitStatus to EXIT_STATUS_MEMORY_FAILURE + * + * @param frame + * @return whether the frame has been updated + */ + boolean updateFrameMemoryError(FrameInterface frame); + /** * Update Memory usage data and LLU time for the given frame. * diff --git a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/DispatchSupportService.java b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/DispatchSupportService.java index 887d0c29e..713f6c86c 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/DispatchSupportService.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/DispatchSupportService.java @@ -42,6 +42,7 @@ import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.LogManager; import org.springframework.dao.EmptyResultDataAccessException; +import org.springframework.dao.DataAccessException; import org.springframework.transaction.annotation.Propagation; import org.springframework.transaction.annotation.Transactional; @@ -184,7 +185,11 @@ public boolean increaseReservedMemory(ProcInterface p, long value) { @Override public boolean clearVirtualProcAssignement(ProcInterface proc) { - return procDao.clearVirtualProcAssignment(proc); + try { + return procDao.clearVirtualProcAssignment(proc); + } catch (DataAccessException e) { + return false; + } } @Transactional(propagation = Propagation.REQUIRED) @@ -343,6 +348,12 @@ public void clearFrame(DispatchFrame frame) { frameDao.updateFrameCleared(frame); } + @Override + @Transactional(propagation = Propagation.REQUIRED) + public boolean updateFrameMemoryError(FrameInterface frame) { + return frameDao.updateFrameMemoryError(frame); + } + @Transactional(propagation = Propagation.SUPPORTS) public RunFrame prepareRqdRunFrame(VirtualProc proc, DispatchFrame frame) { int threads = proc.coresReserved / 100; diff --git a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/Dispatcher.java b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/Dispatcher.java index 6ac703a41..072b04113 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/Dispatcher.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/Dispatcher.java @@ -1,4 +1,3 @@ - /* * Copyright Contributors to the OpenCue Project * @@ -108,13 +107,8 @@ public interface Dispatcher { // without being penalized for it. public static final long VIRTUAL_MEM_THRESHHOLD = CueUtil.GB2; - // The amount of swap that must be used before a host can go - // into kill mode. - public static final long KILL_MODE_SWAP_THRESHOLD = CueUtil.MB128; - - // When the amount of free memory drops below this point, the - // host can go into kill mode. - public static final long KILL_MODE_MEM_THRESHOLD = CueUtil.MB512; + // How long to keep track of a frame kill request + public static final int FRAME_KILL_CACHE_EXPIRE_AFTER_WRITE_MINUTES = 3; // A higher number gets more deep booking but less spread on the cue. public static final int DEFAULT_MAX_FRAMES_PER_PASS = 4; diff --git a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/FrameCompleteHandler.java b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/FrameCompleteHandler.java index 55336aaf4..c405a9e31 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/FrameCompleteHandler.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/FrameCompleteHandler.java @@ -33,6 +33,7 @@ import com.imageworks.spcue.DispatchFrame; import com.imageworks.spcue.DispatchHost; import com.imageworks.spcue.DispatchJob; +import com.imageworks.spcue.FrameDetail; import com.imageworks.spcue.JobDetail; import com.imageworks.spcue.LayerDetail; import com.imageworks.spcue.LayerInterface; @@ -143,49 +144,35 @@ public void handleFrameCompleteReport(final FrameCompleteReport report) { } try { - - final VirtualProc proc; - - try { - - proc = hostManager.getVirtualProc( - report.getFrame().getResourceId()); - } - catch (EmptyResultDataAccessException e) { - /* - * Do not propagate this exception to RQD. This - * usually means the cue lost connectivity to - * the host and cleared out the record of the proc. - * If this is propagated back to RQD, RQD will - * keep retrying the operation forever. - */ - logger.info("failed to acquire data needed to " + - "process completed frame: " + - report.getFrame().getFrameName() + " in job " + - report.getFrame().getJobName() + "," + e); - return; - } - + final VirtualProc proc = hostManager.getVirtualProc(report.getFrame().getResourceId()); final DispatchJob job = jobManager.getDispatchJob(proc.getJobId()); final LayerDetail layer = jobManager.getLayerDetail(report.getFrame().getLayerId()); + final FrameDetail frameDetail = jobManager.getFrameDetail(report.getFrame().getFrameId()); final DispatchFrame frame = jobManager.getDispatchFrame(report.getFrame().getFrameId()); final FrameState newFrameState = determineFrameState(job, layer, frame, report); final String key = proc.getJobId() + "_" + report.getFrame().getLayerId() + "_" + report.getFrame().getFrameId(); + if (dispatchSupport.stopFrame(frame, newFrameState, report.getExitStatus(), report.getFrame().getMaxRss())) { - dispatchQueue.execute(new KeyRunnable(key) { - @Override - public void run() { - try { - handlePostFrameCompleteOperations(proc, report, job, frame, - newFrameState); - } catch (Exception e) { - logger.warn("Exception during handlePostFrameCompleteOperations " + - "in handleFrameCompleteReport" + CueExceptionUtil.getStackTrace(e)); + if (dispatcher.isTestMode()) { + // Database modifications on a threadpool cannot be captured by the test thread + handlePostFrameCompleteOperations(proc, report, job, frame, + newFrameState, frameDetail); + } else { + dispatchQueue.execute(new KeyRunnable(key) { + @Override + public void run() { + try { + handlePostFrameCompleteOperations(proc, report, job, frame, + newFrameState, frameDetail); + } catch (Exception e) { + logger.warn("Exception during handlePostFrameCompleteOperations " + + "in handleFrameCompleteReport" + CueExceptionUtil.getStackTrace(e)); + } } - } - }); + }); + } } else { /* @@ -222,6 +209,19 @@ public void run() { } } } + catch (EmptyResultDataAccessException e) { + /* + * Do not propagate this exception to RQD. This + * usually means the cue lost connectivity to + * the host and cleared out the record of the proc. + * If this is propagated back to RQD, RQD will + * keep retrying the operation forever. + */ + logger.info("failed to acquire data needed to " + + "process completed frame: " + + report.getFrame().getFrameName() + " in job " + + report.getFrame().getJobName() + "," + e); + } catch (Exception e) { /* @@ -259,7 +259,7 @@ public void run() { */ public void handlePostFrameCompleteOperations(VirtualProc proc, FrameCompleteReport report, DispatchJob job, DispatchFrame frame, - FrameState newFrameState) { + FrameState newFrameState, FrameDetail frameDetail) { try { /* @@ -313,7 +313,8 @@ public void handlePostFrameCompleteOperations(VirtualProc proc, * specified in the show's service override, service or 2GB. */ if (report.getExitStatus() == Dispatcher.EXIT_STATUS_MEMORY_FAILURE - || report.getExitSignal() == Dispatcher.EXIT_STATUS_MEMORY_FAILURE) { + || report.getExitSignal() == Dispatcher.EXIT_STATUS_MEMORY_FAILURE + || frameDetail.exitStatus == Dispatcher.EXIT_STATUS_MEMORY_FAILURE) { long increase = CueUtil.GB2; // since there can be multiple services, just going for the diff --git a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/HostReportHandler.java b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/HostReportHandler.java index 2adef34fb..997e32fd4 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/HostReportHandler.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/HostReportHandler.java @@ -21,12 +21,16 @@ import java.sql.Timestamp; import java.util.ArrayList; -import java.util.EnumSet; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.concurrent.ExecutionException; import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import com.google.common.cache.Cache; +import com.google.common.cache.CacheBuilder; +import com.imageworks.spcue.JobInterface; import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.LogManager; import org.springframework.beans.factory.annotation.Autowired; @@ -50,6 +54,7 @@ import com.imageworks.spcue.dispatcher.commands.DispatchBookHostLocal; import com.imageworks.spcue.dispatcher.commands.DispatchHandleHostReport; import com.imageworks.spcue.dispatcher.commands.DispatchRqdKillFrame; +import com.imageworks.spcue.dispatcher.commands.DispatchRqdKillFrameMemory; import com.imageworks.spcue.grpc.host.HardwareState; import com.imageworks.spcue.grpc.host.LockState; import com.imageworks.spcue.grpc.report.BootReport; @@ -63,10 +68,11 @@ import com.imageworks.spcue.service.CommentManager; import com.imageworks.spcue.service.HostManager; import com.imageworks.spcue.service.JobManager; -import com.imageworks.spcue.service.JobManagerSupport; import com.imageworks.spcue.util.CueExceptionUtil; import com.imageworks.spcue.util.CueUtil; +import static com.imageworks.spcue.dispatcher.Dispatcher.*; + public class HostReportHandler { private static final Logger logger = LogManager.getLogger(HostReportHandler.class); @@ -81,7 +87,6 @@ public class HostReportHandler { private Dispatcher localDispatcher; private RqdClient rqdClient; private JobManager jobManager; - private JobManagerSupport jobManagerSupport; private JobDao jobDao; private LayerDao layerDao; @Autowired @@ -93,6 +98,13 @@ public class HostReportHandler { "space on the temporary directory (mcp)"; private static final String CUEBOT_COMMENT_USER = "cuebot"; + // A cache to store kill requests and count the number of occurrences. + // The cache expires after write to avoid growing unbounded. If a request for a host-frame doesn't appear + // for a period of time, the entry will be removed. + Cache killRequestCounterCache = CacheBuilder.newBuilder() + .expireAfterWrite(FRAME_KILL_CACHE_EXPIRE_AFTER_WRITE_MINUTES, TimeUnit.MINUTES) + .build(); + /** * Boolean to toggle if this class is accepting data or not. */ @@ -143,7 +155,6 @@ public void queueHostReport(HostReport report) { reportQueue.execute(new DispatchHandleHostReport(report, this)); } - public void handleHostReport(HostReport report, boolean isBoot) { long startTime = System.currentTimeMillis(); try { @@ -211,9 +222,9 @@ public void handleHostReport(HostReport report, boolean isBoot) { killTimedOutFrames(report); /* - * Increase/decreased reserved memory. + * Prevent OOM (Out-Of-Memory) issues on the host and manage frame reserved memory */ - handleMemoryReservations(host, report); + handleMemoryUsage(host, report); /* * The checks are done in order of least CPU intensive to @@ -456,103 +467,279 @@ private void changeLockState(DispatchHost host, CoreDetail coreInfo) { } /** - * Handle memory reservations for the given host. This will re-balance memory - * reservations on the machine and kill and frames that are out of control. - * + * Prevent host from entering an OOM state where oom-killer might start killing important OS processes. + * The kill logic will kick in one of the following conditions is met: + * - Host has less than OOM_MEMORY_LEFT_THRESHOLD_PERCENT memory available + * - A frame is taking more than OOM_FRAME_OVERBOARD_PERCENT of what it had reserved + * For frames that are using more than they had reserved but not above the threshold, negotiate expanding + * the reservations with other frames on the same host * @param host * @param report */ - private void handleMemoryReservations(final DispatchHost host, final HostReport report) { + private void handleMemoryUsage(final DispatchHost host, final HostReport report) { + // Don't keep memory balances on nimby hosts + if (host.isNimby) { + return; + } - // TODO: GPU: Need to keep frames from growing into space reserved for GPU frames - // However all this is done in the database without a chance to edit the values here + final double OOM_MAX_SAFE_USED_MEMORY_THRESHOLD = + env.getRequiredProperty("dispatcher.oom_max_safe_used_memory_threshold", Double.class); + final double OOM_FRAME_OVERBOARD_ALLOWED_THRESHOLD = + env.getRequiredProperty("dispatcher.oom_frame_overboard_allowed_threshold", Double.class); + RenderHost renderHost = report.getHost(); + List runningFrames = report.getFramesList(); + + boolean memoryWarning = renderHost.getTotalMem() > 0 && + ((double)renderHost.getFreeMem()/renderHost.getTotalMem() < + (1.0 - OOM_MAX_SAFE_USED_MEMORY_THRESHOLD)); + + if (memoryWarning) { + long memoryAvailable = renderHost.getFreeMem(); + long minSafeMemoryAvailable = (long)(renderHost.getTotalMem() * (1.0 - OOM_MAX_SAFE_USED_MEMORY_THRESHOLD)); + // Only allow killing up to 10 frames at a time + int killAttemptsRemaining = 10; + VirtualProc killedProc = null; + do { + killedProc = killWorstMemoryOffender(host); + killAttemptsRemaining -= 1; + if (killedProc != null) { + memoryAvailable = memoryAvailable + killedProc.memoryUsed; + } + } while (killAttemptsRemaining > 0 && + memoryAvailable < minSafeMemoryAvailable && + killedProc != null); + } else { + // When no mass cleaning was required, check for frames going overboard + // if frames didn't go overboard, manage its reservations trying to increase + // them accordingly + for (final RunningFrameInfo frame : runningFrames) { + if (OOM_FRAME_OVERBOARD_ALLOWED_THRESHOLD > 0 && isFrameOverboard(frame)) { + if (!killFrameOverusingMemory(frame, host.getName())) { + logger.warn("Frame " + frame.getJobName() + "." + frame.getFrameName() + + " is overboard but could not be killed"); + } + } else { + handleMemoryReservations(frame); + } + } + } + } - /* - * Check to see if we enable kill mode to free up memory. - */ - boolean killMode = hostManager.isSwapping(host); + public enum KillCause { + FrameOverboard("This frame is using more memory than it had reserved."), + HostUnderOom("Frame killed by host under OOM pressure"), + FrameTimedOut("Frame timed out"), + FrameLluTimedOut("Frame LLU timed out"), + FrameVerificationFailure("Frame failed to be verified on the database"); + private final String message; - for (final RunningFrameInfo f: report.getFramesList()) { + private KillCause(String message) { + this.message = message; + } + @Override + public String toString() { + return message; + } + } - VirtualProc proc = null; + private boolean killFrameOverusingMemory(RunningFrameInfo frame, String hostname) { + try { + VirtualProc proc = hostManager.getVirtualProc(frame.getResourceId()); + + // Don't mess with localDispatch procs + if (proc.isLocalDispatch) { + return false; + } + + logger.info("Killing frame on " + frame.getJobName() + "." + frame.getFrameName() + + ", using too much memory."); + return killProcForMemory(proc, hostname, KillCause.FrameOverboard); + } catch (EmptyResultDataAccessException e) { + return false; + } + } + + private boolean getKillClearance(String hostname, String frameId) { + String cacheKey = hostname + "-" + frameId; + final int FRAME_KILL_RETRY_LIMIT = + env.getRequiredProperty("dispatcher.frame_kill_retry_limit", Integer.class); + + // Cache frame+host receiving a killRequest and count how many times the request is being retried + // meaning rqd is probably failing at attempting to kill the related proc + long cachedCount; + try { + cachedCount = 1 + killRequestCounterCache.get(cacheKey, () -> 0L); + } catch (ExecutionException e) { + return false; + } + killRequestCounterCache.put(cacheKey, cachedCount); + if (cachedCount > FRAME_KILL_RETRY_LIMIT) { + FrameInterface frame = jobManager.getFrame(frameId); + JobInterface job = jobManager.getJob(frame.getJobId()); + + logger.warn("KillRequest blocked for " + job.getName() + "." + frame.getName() + + " blocked for host " + hostname + ". The kill retry limit has been reached."); + return false; + } + return true; + } + + private boolean killProcForMemory(VirtualProc proc, String hostname, KillCause killCause) { + if (!getKillClearance(hostname, proc.frameId)) { + return false; + } + + FrameInterface frame = jobManager.getFrame(proc.frameId); + if (dispatcher.isTestMode()) { + // Different threads don't share the same database state on the test environment + (new DispatchRqdKillFrameMemory(hostname, frame, killCause.toString(), rqdClient, + dispatchSupport, dispatcher.isTestMode())).run(); + } else { try { - proc = hostManager.getVirtualProc(f.getResourceId()); + killQueue.execute(new DispatchRqdKillFrameMemory(hostname, frame, killCause.toString(), rqdClient, + dispatchSupport, dispatcher.isTestMode())); + } catch (TaskRejectedException e) { + logger.warn("Unable to add a DispatchRqdKillFrame request, task rejected, " + e); + return false; + } + } + DispatchSupport.killedOffenderProcs.incrementAndGet(); + return true; + } - // TODO: handle memory management for local dispatches - // Skip local dispatches for now. - if (proc.isLocalDispatch) { - continue; - } + private boolean killFrame(String frameId, String hostname, KillCause killCause) { + if (!getKillClearance(hostname, frameId)) { + return false; + } + if (dispatcher.isTestMode()) { + // Different threads don't share the same database state on the test environment + (new DispatchRqdKillFrame(hostname, frameId, killCause.toString(), rqdClient)).run(); + } else { + try { + killQueue.execute(new DispatchRqdKillFrame(hostname, + frameId, + killCause.toString(), + rqdClient)); + } catch (TaskRejectedException e) { + logger.warn("Unable to add a DispatchRqdKillFrame request, task rejected, " + e); + } + } + DispatchSupport.killedOffenderProcs.incrementAndGet(); + return true; + } - if (f.getRss() > host.memory) { - try{ - logger.info("Killing frame " + f.getJobName() + "/" + f.getFrameName() + ", " - + proc.getName() + " was OOM"); - try { - killQueue.execute(new DispatchRqdKillFrame(proc, "The frame required " + - CueUtil.KbToMb(f.getRss()) + " but the machine only has " + - CueUtil.KbToMb(host.memory), rqdClient)); - } catch (TaskRejectedException e) { - logger.warn("Unable to queue RQD kill, task rejected, " + e); - } - DispatchSupport.killedOomProcs.incrementAndGet(); - } catch (Exception e) { - logger.info("failed to kill frame on " + proc.getName() + - "," + e); - } - } + /** + * Kill proc with the worst user/reserved memory ratio. + * + * @param host + * @return killed proc, or null if none could be found or failed to be killed + */ + private VirtualProc killWorstMemoryOffender(final DispatchHost host) { + try { + VirtualProc proc = hostManager.getWorstMemoryOffender(host); + logger.info("Killing frame on " + proc.getName() + ", host is under stress."); - if (dispatchSupport.increaseReservedMemory(proc, f.getRss())) { - proc.memoryReserved = f.getRss(); - logger.info("frame " + f.getFrameName() + " on job " + f.getJobName() - + " increased its reserved memory to " + - CueUtil.KbToMb(f.getRss())); - } + if (!killProcForMemory(proc, host.getName(), KillCause.HostUnderOom)) { + proc = null; + } + return proc; + } + catch (EmptyResultDataAccessException e) { + logger.error(host.name + " is under OOM and no proc is running on it."); + return null; + } + } + + /** + * Check frame memory usage comparing the amount used with the amount it had reserved + * @param frame + * @return + */ + private boolean isFrameOverboard(final RunningFrameInfo frame) { + final double OOM_FRAME_OVERBOARD_ALLOWED_THRESHOLD = + env.getRequiredProperty("dispatcher.oom_frame_overboard_allowed_threshold", Double.class); + + if (OOM_FRAME_OVERBOARD_ALLOWED_THRESHOLD < 0) { + return false; + } + + double rss = (double)frame.getRss(); + double maxRss = (double)frame.getMaxRss(); + final double MAX_RSS_OVERBOARD_THRESHOLD = OOM_FRAME_OVERBOARD_ALLOWED_THRESHOLD * 2; + final double RSS_AVAILABLE_FOR_MAX_RSS_TRIGGER = 0.1; + + try { + VirtualProc proc = hostManager.getVirtualProc(frame.getResourceId()); + double reserved = (double)proc.memoryReserved; + + // Last memory report is higher than the threshold + if (isOverboard(rss, reserved, OOM_FRAME_OVERBOARD_ALLOWED_THRESHOLD)) { + return true; + } + // If rss is not overboard, handle the situation where the frame might be going overboard from + // time to time but the last report wasn't during a spike. For this case, consider a combination + // of rss and maxRss. maxRss > 2 * threshold and rss > 0.9 + else { + return isOverboard(maxRss, reserved, MAX_RSS_OVERBOARD_THRESHOLD) && + isOverboard(rss, reserved, -RSS_AVAILABLE_FOR_MAX_RSS_TRIGGER); + } + } catch (EmptyResultDataAccessException e) { + logger.info("HostReportHandler(isFrameOverboard): Virtual proc for frame " + + frame.getFrameName() + " on job " + frame.getJobName() + " doesn't exist on the database"); + // Not able to mark the frame overboard is it couldn't be found on the db. + // Proc accounting (verifyRunningProc) should take care of it + return false; + } + } - } catch (ResourceReservationFailureException e) { + private boolean isOverboard(double value, double total, double threshold) { + return value/total >= (1 + threshold); + } - long memNeeded = f.getRss() - proc.memoryReserved; + /** + * Handle memory reservations for the given frame + * + * @param frame + */ + private void handleMemoryReservations(final RunningFrameInfo frame) { + VirtualProc proc = null; + try { + proc = hostManager.getVirtualProc(frame.getResourceId()); - logger.info("frame " + f.getFrameName() + " on job " + f.getJobName() + if (proc.isLocalDispatch) { + return; + } + + if (dispatchSupport.increaseReservedMemory(proc, frame.getRss())) { + proc.memoryReserved = frame.getRss(); + logger.info("frame " + frame.getFrameName() + " on job " + frame.getJobName() + + " increased its reserved memory to " + + CueUtil.KbToMb(frame.getRss())); + } + } catch (ResourceReservationFailureException e) { + if (proc != null) { + long memNeeded = frame.getRss() - proc.memoryReserved; + logger.info("frame " + frame.getFrameName() + " on job " + frame.getJobName() + "was unable to reserve an additional " + CueUtil.KbToMb(memNeeded) + "on proc " + proc.getName() + ", " + e); - try { if (dispatchSupport.balanceReservedMemory(proc, memNeeded)) { - proc.memoryReserved = f.getRss(); + proc.memoryReserved = frame.getRss(); logger.info("was able to balance host: " + proc.getName()); - } - else { + } else { logger.info("failed to balance host: " + proc.getName()); } } catch (Exception ex) { logger.warn("failed to balance host: " + proc.getName() + ", " + e); } - } catch (EmptyResultDataAccessException e) { - logger.info("HostReportHandler: frame " + f.getFrameName() + - " on job " + f.getJobName() + - " was unable be processed" + - " because the proc could not be found"); - } - } - - if (killMode) { - VirtualProc proc; - try { - proc = hostManager.getWorstMemoryOffender(host); - } - catch (EmptyResultDataAccessException e) { - logger.info(host.name + " is swapping and no proc is running on it."); - return; + } else { + logger.info("frame " + frame.getFrameName() + " on job " + frame.getJobName() + + "was unable to reserve an additional memory. Proc could not be found"); } - - logger.info("Killing frame on " + - proc.getName() + ", host is distressed."); - - DispatchSupport.killedOffenderProcs.incrementAndGet(); - jobManagerSupport.kill(proc, new Source( - "The host was dangerously low on memory and swapping.")); + } catch (EmptyResultDataAccessException e) { + logger.info("HostReportHandler: Memory reservations for frame " + frame.getFrameName() + + " on job " + frame.getJobName() + " proc could not be found"); } } @@ -562,7 +749,6 @@ private void handleMemoryReservations(final DispatchHost host, final HostReport * @param rFrames */ private void killTimedOutFrames(HostReport report) { - final Map layers = new HashMap(5); for (RunningFrameInfo frame: report.getFramesList()) { @@ -570,36 +756,16 @@ private void killTimedOutFrames(HostReport report) { LayerDetail layer = layerDao.getLayerDetail(layerId); long runtimeMinutes = ((System.currentTimeMillis() - frame.getStartTime()) / 1000l) / 60; - if (layer.timeout != 0 && runtimeMinutes > layer.timeout){ - try { - killQueue.execute(new DispatchRqdKillFrame(report.getHost().getName(), - frame.getFrameId(), - "This frame has reached it timeout.", - rqdClient)); - } catch (TaskRejectedException e) { - logger.warn("Unable to queue RQD kill, task rejected, " + e); - } - } + String hostname = report.getHost().getName(); - if (layer.timeout_llu == 0){ - continue; - } - - if (frame.getLluTime() == 0){ - continue; - } - - long r = System.currentTimeMillis() / 1000; - long lastUpdate = (r - frame.getLluTime()) / 60; + if (layer.timeout != 0 && runtimeMinutes > layer.timeout){ + killFrame(frame.getFrameId(), hostname, KillCause.FrameTimedOut); + } else if (layer.timeout_llu != 0 && frame.getLluTime() != 0) { + long r = System.currentTimeMillis() / 1000; + long lastUpdate = (r - frame.getLluTime()) / 60; - if (layer.timeout_llu != 0 && lastUpdate > (layer.timeout_llu -1)){ - try { - killQueue.execute(new DispatchRqdKillFrame(report.getHost().getName(), - frame.getFrameId(), - "This frame has reached it LLU timeout.", - rqdClient)); - } catch (TaskRejectedException e) { - logger.warn("Unable to queue RQD kill, task rejected, " + e); + if (layer.timeout_llu != 0 && lastUpdate > (layer.timeout_llu - 1)){ + killFrame(frame.getFrameId(), hostname, KillCause.FrameLluTimedOut); } } } @@ -725,98 +891,59 @@ public void verifyRunningFrameInfo(HostReport report) { continue; } + if (hostManager.verifyRunningProc(runningFrame.getResourceId(), runningFrame.getFrameId())) { + runningFrames.add(runningFrame); + continue; + } - if (!hostManager.verifyRunningProc(runningFrame.getResourceId(), - runningFrame.getFrameId())) { - - /* - * The frame this proc is running is no longer - * assigned to this proc. Don't ever touch - * the frame record. If we make it here that means - * the proc has been running for over 2 min. - */ - - String msg; - VirtualProc proc = null; + /* + * The frame this proc is running is no longer + * assigned to this proc. Don't ever touch + * the frame record. If we make it here that means + * the proc has been running for over 2 min. + */ + String msg; + VirtualProc proc = null; - try { - proc = hostManager.getVirtualProc(runningFrame.getResourceId()); - msg = "Virutal proc " + proc.getProcId() + + try { + proc = hostManager.getVirtualProc(runningFrame.getResourceId()); + msg = "Virtual proc " + proc.getProcId() + "is assigned to " + proc.getFrameId() + " not " + runningFrame.getFrameId(); - } - catch (Exception e) { - /* - * This will happen if the host goes off line and then - * comes back. In this case, we don't touch the frame - * since it might already be running somewhere else. We - * do however kill the proc. - */ - msg = "Virtual proc did not exist."; - } - - logger.info("warning, the proc " + - runningFrame.getResourceId() + " on host " + - report.getHost().getName() + " was running for " + - (runtimeSeconds / 60.0f) + " minutes " + - runningFrame.getJobName() + "/" + runningFrame.getFrameName() + - "but the DB did not " + - "reflect this " + - msg); - - DispatchSupport.accountingErrors.incrementAndGet(); - - try { - /* - * If the proc did exist unbook it if we can't - * verify its running something. - */ - boolean rqd_kill = false; - if (proc != null) { - - /* - * Check to see if the proc is an orphan. - */ - if (hostManager.isOprhan(proc)) { - dispatchSupport.clearVirtualProcAssignement(proc); - dispatchSupport.unbookProc(proc); - rqd_kill = true; - } - } - else { - /* Proc doesn't exist so a kill won't hurt */ - rqd_kill = true; - } - - if (rqd_kill) { - try { - killQueue.execute(new DispatchRqdKillFrame(report.getHost().getName(), - runningFrame.getFrameId(), - "OpenCue could not verify this frame.", - rqdClient)); - } catch (TaskRejectedException e) { - logger.warn("Unable to queue RQD kill, task rejected, " + e); - } - } + } + catch (Exception e) { + /* + * This will happen if the host goes offline and then + * comes back. In this case, we don't touch the frame + * since it might already be running somewhere else. We + * do however kill the proc. + */ + msg = "Virtual proc did not exist."; + } - } catch (RqdClientException rqde) { - logger.warn("failed to kill " + - runningFrame.getJobName() + "/" + - runningFrame.getFrameName() + - " when trying to clear a failed " + - " frame verification, " + rqde); - - } catch (Exception e) { - CueExceptionUtil.logStackTrace("failed", e); - logger.warn("failed to verify " + - runningFrame.getJobName() + "/" + - runningFrame.getFrameName() + - " was running but the frame was " + - " unable to be killed, " + e); - } + DispatchSupport.accountingErrors.incrementAndGet(); + if (proc != null && hostManager.isOprhan(proc)) { + dispatchSupport.clearVirtualProcAssignement(proc); + dispatchSupport.unbookProc(proc); + proc = null; } - else { - runningFrames.add(runningFrame); + if (proc == null) { + if (killFrame(runningFrame.getFrameId(), + report.getHost().getName(), + KillCause.FrameVerificationFailure)) { + logger.info("FrameVerificationError, the proc " + + runningFrame.getResourceId() + " on host " + + report.getHost().getName() + " was running for " + + (runtimeSeconds / 60.0f) + " minutes " + + runningFrame.getJobName() + "/" + runningFrame.getFrameName() + + "but the DB did not " + + "reflect this. " + + msg); + } else { + logger.warn("FrameStuckWarning: frameId=" + runningFrame.getFrameId() + + " render_node=" + report.getHost().getName() + " - " + + runningFrame.getJobName() + "/" + runningFrame.getFrameName()); + } } } } @@ -877,14 +1004,6 @@ public void setJobManager(JobManager jobManager) { this.jobManager = jobManager; } - public JobManagerSupport getJobManagerSupport() { - return jobManagerSupport; - } - - public void setJobManagerSupport(JobManagerSupport jobManagerSupport) { - this.jobManagerSupport = jobManagerSupport; - } - public JobDao getJobDao() { return jobDao; } diff --git a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/commands/DispatchRqdKillFrame.java b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/commands/DispatchRqdKillFrame.java index 61258a824..fe9bde60e 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/commands/DispatchRqdKillFrame.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/commands/DispatchRqdKillFrame.java @@ -31,9 +31,7 @@ public class DispatchRqdKillFrame extends KeyRunnable { private static final Logger logger = LogManager.getLogger(DispatchRqdKillFrame.class); - private VirtualProc proc = null; private String message; - private String hostname; private String frameId; @@ -47,28 +45,14 @@ public DispatchRqdKillFrame(String hostname, String frameId, String message, Rqd this.rqdClient = rqdClient; } - public DispatchRqdKillFrame(VirtualProc proc, String message, RqdClient rqdClient) { - super("disp_rqd_kill_frame_" + proc.getProcId() + "_" + rqdClient.toString()); - this.proc = proc; - this.hostname = proc.hostName; - this.message = message; - this.rqdClient = rqdClient; - } - @Override public void run() { long startTime = System.currentTimeMillis(); try { - if (proc != null) { - rqdClient.killFrame(proc, message); - } - else { - rqdClient.killFrame(hostname, frameId, message); - } + rqdClient.killFrame(hostname, frameId, message); } catch (RqdClientException e) { logger.info("Failed to contact host " + hostname + ", " + e); - } - finally { + } finally { long elapsedTime = System.currentTimeMillis() - startTime; logger.info("RQD communication with " + hostname + " took " + elapsedTime + "ms"); diff --git a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/commands/DispatchRqdKillFrameMemory.java b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/commands/DispatchRqdKillFrameMemory.java new file mode 100644 index 000000000..301f77479 --- /dev/null +++ b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/commands/DispatchRqdKillFrameMemory.java @@ -0,0 +1,78 @@ + +/* + * Copyright Contributors to the OpenCue Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + + +package com.imageworks.spcue.dispatcher.commands; + +import com.imageworks.spcue.FrameInterface; +import com.imageworks.spcue.VirtualProc; +import com.imageworks.spcue.dispatcher.DispatchSupport; +import com.imageworks.spcue.rqd.RqdClient; +import com.imageworks.spcue.rqd.RqdClientException; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.LogManager; + +/** + * A runnable to communicate with rqd requesting for a frame to be killed due to memory issues. + *

+ * Before killing a frame, the database is updated to mark the frame status as EXIT_STATUS_MEMORY_FAILURE, + * this allows the FrameCompleteHandler to possibly retry the frame after increasing its memory requirements + */ +public class DispatchRqdKillFrameMemory extends KeyRunnable { + + private static final Logger logger = LogManager.getLogger(DispatchRqdKillFrameMemory.class); + + private String message; + private String hostname; + private DispatchSupport dispatchSupport; + private final RqdClient rqdClient; + private final boolean isTestMode; + + private FrameInterface frame; + + public DispatchRqdKillFrameMemory(String hostname, FrameInterface frame, String message, RqdClient rqdClient, + DispatchSupport dispatchSupport, boolean isTestMode) { + super("disp_rqd_kill_frame_" + frame.getFrameId() + "_" + rqdClient.toString()); + this.frame = frame; + this.hostname = hostname; + this.message = message; + this.rqdClient = rqdClient; + this.dispatchSupport = dispatchSupport; + this.isTestMode = isTestMode; + } + + @Override + public void run() { + long startTime = System.currentTimeMillis(); + try { + if (dispatchSupport.updateFrameMemoryError(frame) && !isTestMode) { + rqdClient.killFrame(hostname, frame.getFrameId(), message); + } else { + logger.warn("Could not update frame " + frame.getFrameId() + + " status to EXIT_STATUS_MEMORY_FAILURE. Canceling kill request!"); + } + } catch (RqdClientException e) { + logger.warn("Failed to contact host " + hostname + ", " + e); + } finally { + long elapsedTime = System.currentTimeMillis() - startTime; + logger.info("RQD communication with " + hostname + + " took " + elapsedTime + "ms"); + } + } +} + diff --git a/cuebot/src/main/java/com/imageworks/spcue/service/HostManager.java b/cuebot/src/main/java/com/imageworks/spcue/service/HostManager.java index aaf401688..e62d8647b 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/service/HostManager.java +++ b/cuebot/src/main/java/com/imageworks/spcue/service/HostManager.java @@ -70,15 +70,6 @@ public interface HostManager { */ void setHostFreeTempDir(HostInterface host, Long freeTempDir); - /** - * Return true if the host is swapping hard enough - * that killing frames will save the entire machine. - * - * @param host - * @return - */ - boolean isSwapping(HostInterface host); - DispatchHost createHost(HostReport report); DispatchHost createHost(RenderHost host); diff --git a/cuebot/src/main/java/com/imageworks/spcue/service/HostManagerService.java b/cuebot/src/main/java/com/imageworks/spcue/service/HostManagerService.java index a1533d695..36de34a1c 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/service/HostManagerService.java +++ b/cuebot/src/main/java/com/imageworks/spcue/service/HostManagerService.java @@ -98,12 +98,6 @@ public void setHostFreeTempDir(HostInterface host, Long freeTempDir) { hostDao.updateHostFreeTempDir(host, freeTempDir); } - @Override - @Transactional(propagation = Propagation.REQUIRED, readOnly=true) - public boolean isSwapping(HostInterface host) { - return hostDao.isKillMode(host); - } - public void rebootWhenIdle(HostInterface host) { try { hostDao.updateHostState(host, HardwareState.REBOOT_WHEN_IDLE); diff --git a/cuebot/src/main/resources/conf/spring/applicationContext-service.xml b/cuebot/src/main/resources/conf/spring/applicationContext-service.xml index 35bd5cbb8..5aedc91b3 100644 --- a/cuebot/src/main/resources/conf/spring/applicationContext-service.xml +++ b/cuebot/src/main/resources/conf/spring/applicationContext-service.xml @@ -384,7 +384,6 @@ - diff --git a/cuebot/src/main/resources/opencue.properties b/cuebot/src/main/resources/opencue.properties index 6b2875899..b7f2a23ff 100644 --- a/cuebot/src/main/resources/opencue.properties +++ b/cuebot/src/main/resources/opencue.properties @@ -130,7 +130,21 @@ dispatcher.booking_queue.max_pool_size=6 # Queue capacity for booking. dispatcher.booking_queue.queue_capacity=1000 -# Whether or not to satisfy dependents (*_ON_FRAME and *_ON_LAYER) only on Frame success +# Percentage of used memory to consider a risk for triggering oom-killer +dispatcher.oom_max_safe_used_memory_threshold=0.95 + +# How much can a frame exceed its reserved memory. +# - 0.5 means 50% above reserve +# - -1.0 makes the feature inactive +# This feature is being kept inactive for now as we work on improving the +# frame retry logic (See commit comment for more details). +dispatcher.oom_frame_overboard_allowed_threshold=-1.0 + +# How many times should cuebot send a kill request for the same frame-host before reporting +# the frame as stuck +dispatcher.frame_kill_retry_limit=3 + +# Whether to satisfy dependents (*_ON_FRAME and *_ON_LAYER) only on Frame success depend.satisfy_only_on_frame_success=true # Jobs will be archived to the history tables after being completed for this long. diff --git a/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/HostDaoTests.java b/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/HostDaoTests.java index a6261e464..918b43679 100644 --- a/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/HostDaoTests.java +++ b/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/HostDaoTests.java @@ -331,24 +331,6 @@ public void testIsHostLocked() { assertEquals(hostDao.isHostLocked(host),true); } - @Test - @Transactional - @Rollback(true) - public void testIsKillMode() { - hostDao.insertRenderHost(buildRenderHost(TEST_HOST), - hostManager.getDefaultAllocationDetail(), - false); - - HostEntity host = hostDao.findHostDetail(TEST_HOST); - assertFalse(hostDao.isKillMode(host)); - - jdbcTemplate.update( - "UPDATE host_stat SET int_swap_free = ?, int_mem_free = ? WHERE pk_host = ?", - CueUtil.MB256, CueUtil.MB256, host.getHostId()); - - assertTrue(hostDao.isKillMode(host)); - } - @Test @Transactional @Rollback(true) diff --git a/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/FrameCompleteHandlerTests.java b/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/FrameCompleteHandlerTests.java index d313c5293..1f452e92a 100644 --- a/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/FrameCompleteHandlerTests.java +++ b/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/FrameCompleteHandlerTests.java @@ -310,10 +310,11 @@ private void executeDepend( DispatchJob dispatchJob = jobManager.getDispatchJob(proc.getJobId()); DispatchFrame dispatchFrame = jobManager.getDispatchFrame(report.getFrame().getFrameId()); + FrameDetail frameDetail = jobManager.getFrameDetail(report.getFrame().getFrameId()); dispatchSupport.stopFrame(dispatchFrame, frameState, report.getExitStatus(), report.getFrame().getMaxRss()); frameCompleteHandler.handlePostFrameCompleteOperations(proc, - report, dispatchJob, dispatchFrame, frameState); + report, dispatchJob, dispatchFrame, frameState, frameDetail); assertTrue(jobManager.isLayerComplete(layerFirst)); assertFalse(jobManager.isLayerComplete(layerSecond)); @@ -401,10 +402,11 @@ private void executeMinMemIncrease(int expected, boolean override) { DispatchJob dispatchJob = jobManager.getDispatchJob(proc.getJobId()); DispatchFrame dispatchFrame = jobManager.getDispatchFrame(report.getFrame().getFrameId()); + FrameDetail frameDetail = jobManager.getFrameDetail(report.getFrame().getFrameId()); dispatchSupport.stopFrame(dispatchFrame, FrameState.DEAD, report.getExitStatus(), report.getFrame().getMaxRss()); frameCompleteHandler.handlePostFrameCompleteOperations(proc, - report, dispatchJob, dispatchFrame, FrameState.WAITING); + report, dispatchJob, dispatchFrame, FrameState.WAITING, frameDetail); assertFalse(jobManager.isLayerComplete(layer)); diff --git a/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/HostReportHandlerGpuTests.java b/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/HostReportHandlerGpuTests.java index ce1e98ae1..120c620a1 100644 --- a/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/HostReportHandlerGpuTests.java +++ b/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/HostReportHandlerGpuTests.java @@ -83,12 +83,12 @@ private static RenderHost getRenderHost() { .setBootTime(1192369572) // The minimum amount of free space in the temporary directory to book a host. .setFreeMcp(CueUtil.GB) - .setFreeMem(53500) - .setFreeSwap(20760) + .setFreeMem(CueUtil.GB8) + .setFreeSwap(CueUtil.GB2) .setLoad(0) .setTotalMcp(CueUtil.GB4) - .setTotalMem(1048576L * 4096) - .setTotalSwap(20960) + .setTotalMem(CueUtil.GB8) + .setTotalSwap(CueUtil.GB2) .setNimbyEnabled(false) .setNumProcs(2) .setCoresPerProc(100) @@ -115,7 +115,7 @@ public void testHandleHostReport() { hostReportHandler.handleHostReport(report, true); DispatchHost host = getHost(); assertEquals(host.lockState, LockState.OPEN); - assertEquals(host.memory, 4294443008L); + assertEquals(host.memory, CueUtil.GB8 - 524288); assertEquals(host.gpus, 64); assertEquals(host.idleGpus, 64); assertEquals(host.gpuMemory, 1048576L * 2048); diff --git a/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/HostReportHandlerTests.java b/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/HostReportHandlerTests.java index 970b97a95..971df8d14 100644 --- a/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/HostReportHandlerTests.java +++ b/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/HostReportHandlerTests.java @@ -21,10 +21,15 @@ import java.io.File; import java.sql.Timestamp; +import java.util.Arrays; import java.util.List; import java.util.concurrent.ThreadPoolExecutor; import javax.annotation.Resource; +import com.imageworks.spcue.dispatcher.DispatchSupport; +import com.imageworks.spcue.dispatcher.HostReportQueue; +import com.imageworks.spcue.dispatcher.FrameCompleteHandler; +import com.imageworks.spcue.grpc.job.FrameState; import org.junit.Before; import org.junit.Test; import org.springframework.test.annotation.Rollback; @@ -44,6 +49,7 @@ import com.imageworks.spcue.grpc.report.HostReport; import com.imageworks.spcue.grpc.report.RenderHost; import com.imageworks.spcue.grpc.report.RunningFrameInfo; +import com.imageworks.spcue.grpc.report.FrameCompleteReport; import com.imageworks.spcue.service.AdminManager; import com.imageworks.spcue.service.CommentManager; import com.imageworks.spcue.service.HostManager; @@ -52,6 +58,7 @@ import com.imageworks.spcue.test.TransactionalTest; import com.imageworks.spcue.util.CueUtil; import com.imageworks.spcue.VirtualProc; +import com.imageworks.spcue.LayerDetail; import java.util.UUID; @@ -70,6 +77,9 @@ public class HostReportHandlerTests extends TransactionalTest { @Resource HostReportHandler hostReportHandler; + @Resource + FrameCompleteHandler frameCompleteHandler; + @Resource Dispatcher dispatcher; @@ -99,9 +109,9 @@ public void setTestMode() { public void createHost() { hostname = UUID.randomUUID().toString().substring(0, 8); hostname2 = UUID.randomUUID().toString().substring(0, 8); - hostManager.createHost(getRenderHost(hostname, HardwareState.UP), + hostManager.createHost(getRenderHost(hostname), adminManager.findAllocationDetail("spi","general")); - hostManager.createHost(getRenderHost(hostname2, HardwareState.UP), + hostManager.createHost(getRenderHost(hostname2), adminManager.findAllocationDetail("spi","general")); } @@ -118,52 +128,32 @@ private DispatchHost getHost(String hostname) { return hostManager.findDispatchHost(hostname); } - private static RenderHost getRenderHost(String hostname, HardwareState state) { + private static RenderHost.Builder getRenderHostBuilder(String hostname) { return RenderHost.newBuilder() .setName(hostname) .setBootTime(1192369572) // The minimum amount of free space in the temporary directory to book a host. .setFreeMcp(CueUtil.GB) - .setFreeMem((int) CueUtil.GB8) - .setFreeSwap(20760) + .setFreeMem(CueUtil.GB8) + .setFreeSwap(CueUtil.GB2) .setLoad(0) .setTotalMcp(CueUtil.GB4) .setTotalMem(CueUtil.GB8) .setTotalSwap(CueUtil.GB2) .setNimbyEnabled(false) - .setNumProcs(2) + .setNumProcs(16) .setCoresPerProc(100) .addTags("test") - .setState(state) + .setState(HardwareState.UP) .setFacility("spi") .putAttributes("SP_OS", "Linux") - .setFreeGpuMem((int) CueUtil.MB512) - .setTotalGpuMem((int) CueUtil.MB512) - .build(); + .setNumGpus(0) + .setFreeGpuMem(0) + .setTotalGpuMem(0); } - private static RenderHost getRenderHost(String hostname, HardwareState state, Long freeTempDir) { - return RenderHost.newBuilder() - .setName(hostname) - .setBootTime(1192369572) - // The minimum amount of free space in the temporary directory to book a host. - .setFreeMcp(freeTempDir) - .setFreeMem((int) CueUtil.GB8) - .setFreeSwap(20760) - .setLoad(0) - .setTotalMcp(freeTempDir * 4) - .setTotalMem(CueUtil.GB8) - .setTotalSwap(CueUtil.GB2) - .setNimbyEnabled(false) - .setNumProcs(2) - .setCoresPerProc(100) - .addTags("test") - .setState(state) - .setFacility("spi") - .putAttributes("SP_OS", "Linux") - .setFreeGpuMem((int) CueUtil.MB512) - .setTotalGpuMem((int) CueUtil.MB512) - .build(); + private static RenderHost getRenderHost(String hostname) { + return getRenderHostBuilder(hostname).build(); } private static RenderHost getNewRenderHost(String tags) { @@ -172,12 +162,12 @@ private static RenderHost getNewRenderHost(String tags) { .setBootTime(1192369572) // The minimum amount of free space in the temporary directory to book a host. .setFreeMcp(CueUtil.GB) - .setFreeMem(53500) - .setFreeSwap(20760) + .setFreeMem(CueUtil.GB8) + .setFreeSwap(CueUtil.GB2) .setLoad(0) - .setTotalMcp(CueUtil.GB4) - .setTotalMem(8173264) - .setTotalSwap(20960) + .setTotalMcp(195430) + .setTotalMem(CueUtil.GB8) + .setTotalSwap(CueUtil.GB2) .setNimbyEnabled(false) .setNumProcs(2) .setCoresPerProc(100) @@ -196,15 +186,15 @@ private static RenderHost getNewRenderHost(String tags) { public void testHandleHostReport() throws InterruptedException { CoreDetail cores = getCoreDetail(200, 200, 0, 0); HostReport report1 = HostReport.newBuilder() - .setHost(getRenderHost(hostname, HardwareState.UP)) + .setHost(getRenderHost(hostname)) .setCoreInfo(cores) .build(); HostReport report2 = HostReport.newBuilder() - .setHost(getRenderHost(hostname2, HardwareState.UP)) + .setHost(getRenderHost(hostname2)) .setCoreInfo(cores) .build(); HostReport report1_2 = HostReport.newBuilder() - .setHost(getRenderHost(hostname, HardwareState.UP)) + .setHost(getRenderHost(hostname)) .setCoreInfo(getCoreDetail(200, 200, 100, 0)) .build(); @@ -314,7 +304,7 @@ public void testHandleHostReportWithFullTemporaryDirectories() { * */ // Create HostReport HostReport report1 = HostReport.newBuilder() - .setHost(getRenderHost(hostname, HardwareState.UP, 1024L)) + .setHost(getRenderHostBuilder(hostname).setFreeMcp(1024L).build()) .setCoreInfo(cores) .build(); // Call handleHostReport() => Create the comment with subject=SUBJECT_COMMENT_FULL_TEMP_DIR and change the @@ -356,7 +346,7 @@ public void testHandleHostReportWithFullTemporaryDirectories() { * */ // Set the host freeTempDir to the minimum size required = 1GB (1048576 KB) HostReport report2 = HostReport.newBuilder() - .setHost(getRenderHost(hostname, HardwareState.UP, 1048576L)) + .setHost(getRenderHostBuilder(hostname).setFreeMcp(CueUtil.GB).build()) .setCoreInfo(cores) .build(); // Call handleHostReport() => Delete the comment with subject=SUBJECT_COMMENT_FULL_TEMP_DIR and change the @@ -397,7 +387,7 @@ public void testHandleHostReportWithHardwareStateRepairNotRelatedToFullTempDir() * */ // Create HostReport HostReport report = HostReport.newBuilder() - .setHost(getRenderHost(hostname, HardwareState.UP, 1048576L)) + .setHost(getRenderHostBuilder(hostname).setFreeMcp(CueUtil.GB).build()) .setCoreInfo(cores) .build(); // Get host @@ -451,7 +441,7 @@ public void testMemoryAndLlu() { .setMaxRss(420000) .build(); HostReport report = HostReport.newBuilder() - .setHost(getRenderHost(hostname, HardwareState.UP)) + .setHost(getRenderHost(hostname)) .setCoreInfo(cores) .addFrames(info) .build(); @@ -462,5 +452,170 @@ public void testMemoryAndLlu() { assertEquals(frame.dateLLU, new Timestamp(now / 1000 * 1000)); assertEquals(420000, frame.maxRss); } + + @Test + @Transactional + @Rollback(true) + public void testMemoryAggressionRss() { + jobLauncher.testMode = true; + dispatcher.setTestMode(true); + + jobLauncher.launch(new File("src/test/resources/conf/jobspec/jobspec_simple.xml")); + + DispatchHost host = getHost(hostname); + List procs = dispatcher.dispatchHost(host); + assertEquals(1, procs.size()); + VirtualProc proc = procs.get(0); + + // 1.6 = 1 + dispatcher.oom_frame_overboard_allowed_threshold + long memoryOverboard = (long) Math.ceil((double) proc.memoryReserved * 1.6); + + // Test rss overboard + RunningFrameInfo info = RunningFrameInfo.newBuilder() + .setJobId(proc.getJobId()) + .setLayerId(proc.getLayerId()) + .setFrameId(proc.getFrameId()) + .setResourceId(proc.getProcId()) + .setRss(memoryOverboard) + .setMaxRss(memoryOverboard) + .build(); + HostReport report = HostReport.newBuilder() + .setHost(getRenderHost(hostname)) + .setCoreInfo(getCoreDetail(200, 200, 0, 0)) + .addFrames(info) + .build(); + + long killCount = DispatchSupport.killedOffenderProcs.get(); + hostReportHandler.handleHostReport(report, false); + assertEquals(killCount + 1, DispatchSupport.killedOffenderProcs.get()); + } + + @Test + @Transactional + @Rollback(true) + public void testMemoryAggressionMaxRss() { + jobLauncher.testMode = true; + dispatcher.setTestMode(true); + jobLauncher.launch(new File("src/test/resources/conf/jobspec/jobspec_simple.xml")); + + DispatchHost host = getHost(hostname); + List procs = dispatcher.dispatchHost(host); + assertEquals(1, procs.size()); + VirtualProc proc = procs.get(0); + + // 0.6 = dispatcher.oom_frame_overboard_allowed_threshold + long memoryOverboard = (long) Math.ceil((double) proc.memoryReserved * + (1.0 + (2 * 0.6))); + + // Test rss>90% and maxRss overboard + RunningFrameInfo info = RunningFrameInfo.newBuilder() + .setJobId(proc.getJobId()) + .setLayerId(proc.getLayerId()) + .setFrameId(proc.getFrameId()) + .setResourceId(proc.getProcId()) + .setRss((long)Math.ceil(0.95 * proc.memoryReserved)) + .setMaxRss(memoryOverboard) + .build(); + HostReport report = HostReport.newBuilder() + .setHost(getRenderHost(hostname)) + .setCoreInfo(getCoreDetail(200, 200, 0, 0)) + .addFrames(info) + .build(); + + long killCount = DispatchSupport.killedOffenderProcs.get(); + hostReportHandler.handleHostReport(report, false); + assertEquals(killCount + 1, DispatchSupport.killedOffenderProcs.get()); + } + + @Test + @Transactional + @Rollback(true) + public void testMemoryAggressionMemoryWarning() { + jobLauncher.testMode = true; + dispatcher.setTestMode(true); + jobLauncher.launch(new File("src/test/resources/conf/jobspec/jobspec_multiple_frames.xml")); + + DispatchHost host = getHost(hostname); + List procs = dispatcher.dispatchHost(host); + assertEquals(3, procs.size()); + VirtualProc proc1 = procs.get(0); + VirtualProc proc2 = procs.get(1); + VirtualProc proc3 = procs.get(2); + + // Ok + RunningFrameInfo info1 = RunningFrameInfo.newBuilder() + .setJobId(proc1.getJobId()) + .setLayerId(proc1.getLayerId()) + .setFrameId(proc1.getFrameId()) + .setResourceId(proc1.getProcId()) + .setRss(CueUtil.GB2) + .setMaxRss(CueUtil.GB2) + .build(); + + // Overboard Rss + RunningFrameInfo info2 = RunningFrameInfo.newBuilder() + .setJobId(proc2.getJobId()) + .setLayerId(proc2.getLayerId()) + .setFrameId(proc2.getFrameId()) + .setResourceId(proc2.getProcId()) + .setRss(CueUtil.GB4) + .setMaxRss(CueUtil.GB4) + .build(); + + // Overboard Rss + long memoryUsedProc3 = CueUtil.GB8; + RunningFrameInfo info3 = RunningFrameInfo.newBuilder() + .setJobId(proc3.getJobId()) + .setLayerId(proc3.getLayerId()) + .setFrameId(proc3.getFrameId()) + .setResourceId(proc3.getProcId()) + .setRss(memoryUsedProc3) + .setMaxRss(memoryUsedProc3) + .build(); + + RenderHost hostAfterUpdate = getRenderHostBuilder(hostname).setFreeMem(0).build(); + + HostReport report = HostReport.newBuilder() + .setHost(hostAfterUpdate) + .setCoreInfo(getCoreDetail(200, 200, 0, 0)) + .addAllFrames(Arrays.asList(info1, info2, info3)) + .build(); + + // Get layer state before report gets sent + LayerDetail layerBeforeIncrease = jobManager.getLayerDetail(proc3.getLayerId()); + + // In this case, killing one job should be enough to ge the machine to a safe state + long killCount = DispatchSupport.killedOffenderProcs.get(); + hostReportHandler.handleHostReport(report, false); + assertEquals(killCount + 1, DispatchSupport.killedOffenderProcs.get()); + + // Confirm the frame will be set to retry after it's completion has been processed + + RunningFrameInfo runningFrame = RunningFrameInfo.newBuilder() + .setFrameId(proc3.getFrameId()) + .setFrameName("frame_name") + .setLayerId(proc3.getLayerId()) + .setRss(memoryUsedProc3) + .setMaxRss(memoryUsedProc3) + .setResourceId(proc3.id) + .build(); + FrameCompleteReport completeReport = FrameCompleteReport.newBuilder() + .setHost(hostAfterUpdate) + .setFrame(runningFrame) + .setExitSignal(9) + .setRunTime(1) + .setExitStatus(1) + .build(); + + frameCompleteHandler.handleFrameCompleteReport(completeReport); + FrameDetail killedFrame = jobManager.getFrameDetail(proc3.getFrameId()); + LayerDetail layer = jobManager.getLayerDetail(proc3.getLayerId()); + assertEquals(FrameState.WAITING, killedFrame.state); + // Memory increases are processed in two different places one will set the new value to proc.reserved + 2GB + // and the other will set to the maximum reported proc.maxRss the end value will be whoever is higher. + // In this case, proc.maxRss + assertEquals(Math.max(memoryUsedProc3, layerBeforeIncrease.getMinimumMemory() + CueUtil.GB2), + layer.getMinimumMemory()); + } } diff --git a/cuebot/src/test/resources/conf/jobspec/jobspec_multiple_frames.xml b/cuebot/src/test/resources/conf/jobspec/jobspec_multiple_frames.xml new file mode 100644 index 000000000..3baa0b22b --- /dev/null +++ b/cuebot/src/test/resources/conf/jobspec/jobspec_multiple_frames.xml @@ -0,0 +1,48 @@ + + + + + + + + + spi + pipe + default + testuser + 9860 + + + False + 2 + False + + + + echo hello + 1-3 + 1 + 2gb + + + shell + + + + + + diff --git a/cuebot/src/test/resources/opencue.properties b/cuebot/src/test/resources/opencue.properties index 00d0c4463..10516f881 100644 --- a/cuebot/src/test/resources/opencue.properties +++ b/cuebot/src/test/resources/opencue.properties @@ -38,7 +38,7 @@ dispatcher.job_query_max=20 dispatcher.job_lock_expire_seconds=2 dispatcher.job_lock_concurrency_level=3 dispatcher.frame_query_max=10 -dispatcher.job_frame_dispatch_max=2 +dispatcher.job_frame_dispatch_max=3 dispatcher.host_frame_dispatch_max=12 dispatcher.launch_queue.core_pool_size=1 @@ -64,9 +64,8 @@ dispatcher.kill_queue.queue_capacity=1000 dispatcher.booking_queue.core_pool_size=6 dispatcher.booking_queue.max_pool_size=6 dispatcher.booking_queue.queue_capacity=1000 - -# The minimum amount of free space in the temporary directory (mcp) to book a host. -# E.g: 1G = 1048576 kB => dispatcher.min_bookable_free_temp_dir_kb=1048576 -# Default = 1G = 1048576 kB -# If equals to -1, it means the feature is turned off -dispatcher.min_bookable_free_temp_dir_kb=1048576 \ No newline at end of file +dispatcher.min_bookable_free_temp_dir_kb=1048576 +dispatcher.min_bookable_free_mcp_kb=1048576 +dispatcher.oom_max_safe_used_memory_threshold=0.95 +dispatcher.oom_frame_overboard_allowed_threshold=0.6 +dispatcher.frame_kill_retry_limit=3 \ No newline at end of file diff --git a/rqd/rqd/rqdservicers.py b/rqd/rqd/rqdservicers.py index 98ab358ac..b736ef43b 100644 --- a/rqd/rqd/rqdservicers.py +++ b/rqd/rqd/rqdservicers.py @@ -67,6 +67,8 @@ def KillRunningFrame(self, request, context): frame = self.rqCore.getRunningFrame(request.frame_id) if frame: frame.kill(message=request.message) + else: + log.warning("Wasn't able to find frame(%s) to kill", request.frame_id) return rqd.compiled_proto.rqd_pb2.RqdStaticKillRunningFrameResponse() def ShutdownRqdNow(self, request, context): diff --git a/rqd/rqd/rqnetwork.py b/rqd/rqd/rqnetwork.py index 9f33e64a2..9d3591fdd 100644 --- a/rqd/rqd/rqnetwork.py +++ b/rqd/rqd/rqnetwork.py @@ -162,6 +162,7 @@ def kill(self, message=""): else: os.killpg(self.pid, rqd.rqconstants.KILL_SIGNAL) finally: + log.warning("kill() successfully killed frameId=%s pid=%s", self.frameId, self.pid) rqd.rqutil.permissionsLow() except OSError as e: log.warning( From 4e87140ed592e0689ef8597782268916706f17b9 Mon Sep 17 00:00:00 2001 From: Brian Cipriano Date: Wed, 8 Nov 2023 19:00:20 -0500 Subject: [PATCH 43/56] Fix lint warning. (#1329) --- rqd/rqd/rqnetwork.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/rqd/rqd/rqnetwork.py b/rqd/rqd/rqnetwork.py index 9d3591fdd..6bb26cfce 100644 --- a/rqd/rqd/rqnetwork.py +++ b/rqd/rqd/rqnetwork.py @@ -162,7 +162,8 @@ def kill(self, message=""): else: os.killpg(self.pid, rqd.rqconstants.KILL_SIGNAL) finally: - log.warning("kill() successfully killed frameId=%s pid=%s", self.frameId, self.pid) + log.warning( + "kill() successfully killed frameId=%s pid=%s", self.frameId, self.pid) rqd.rqutil.permissionsLow() except OSError as e: log.warning( From fe519e3c377b0d10477bb20740aae087aaf0da35 Mon Sep 17 00:00:00 2001 From: Brian Cipriano Date: Wed, 8 Nov 2023 20:24:10 -0500 Subject: [PATCH 44/56] TSC meeting notes. (#1328) --- tsc/meetings/2023-11-08.md | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) create mode 100644 tsc/meetings/2023-11-08.md diff --git a/tsc/meetings/2023-11-08.md b/tsc/meetings/2023-11-08.md new file mode 100644 index 000000000..07f18796c --- /dev/null +++ b/tsc/meetings/2023-11-08.md @@ -0,0 +1,37 @@ +# OpenCue TSC Meeting Notes 8 Nov 2023 + +Secretary: Brian Cipriano + +Agenda/Notes: + +* OOM protection logic: https://github.com/AcademySoftwareFoundation/OpenCue/pull/1317 + * PR is reviewed and merged. + * Has been running in production for a while now, working well in 90% of cases. + * Other 10% are hitting a race condition with RQD. RQD sends fresh host report after OOM + decision has been made but not actuated yet, clearing the OOM state in the database. Frame is + rescheduled with same memory requirements instead of increased. + * Followup change to RQD coming soon, kill frame with correct OOM code so database state isn't + required. +* RQD env var expansion: https://github.com/AcademySoftwareFoundation/OpenCue/pull/1323 + * Expanding any env vars in the command itself in RQD, before command is executed on host. + * This works already on Linux, Windows doesn't expand vars in command as command written to a + temp file batch script. + * Env vars should be expanded as late as possible, host/RQD env might differ slightly from frame + env. + * Let's move the change into the Windows section. +* RQD copy env vars from host: https://github.com/AcademySoftwareFoundation/OpenCue/pull/1324 + * Needed to preserve the host PYTHONPATH in the frame env. + * Reviewed, change is too broad, left comments on the PR. +* DB indexes: https://github.com/AcademySoftwareFoundation/OpenCue/pull/1304 + * Adding new database indexes to resolve some performance issues. + * Has some commented out pieces, why? + * Diego to check and update PR. +* CUDA RQD image: https://github.com/AcademySoftwareFoundation/OpenCue/pull/1327 + * Let's try to copy RQD package from base image to reduce duplication. + * Should revisit the RQD base image at some point to see if we can upgrade past Python 3.6. +* Blender plugin: https://github.com/AcademySoftwareFoundation/OpenCue/pull/1309 + * Plugin working e2e on Nuwan's machine, getting renders back. + * Starting review of draft PR. + * Would like to see a doc on how to install/use the plugin, will help to understand the code. +* CueGUI PySide compatibility change is rolling out to production soon, working in initial tests but + will get more feedback as usage expands. From b69156fa0ff522aee473c8e4dd2feac2978a7ad9 Mon Sep 17 00:00:00 2001 From: Ramon Figueiredo Date: Mon, 22 Jan 2024 09:08:08 -0800 Subject: [PATCH 45/56] Update Sphinx version (#1336) --- docs/conf.py | 2 +- docs/requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index 2c8322352..230855ca1 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -67,7 +67,7 @@ # # This is also used if you do content translation via gettext catalogs. # Usually you set "language" from the command line for these cases. -language = None +language = 'en' # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. diff --git a/docs/requirements.txt b/docs/requirements.txt index 9d324374d..1f3307af5 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,2 +1,2 @@ -sphinx==4.3.1 +sphinx==5.0.0 sphinx-rtd-theme==1.0.0 From 5b9defd5f5fa87b49270bf7b5851f84066952ed6 Mon Sep 17 00:00:00 2001 From: Brian Cipriano Date: Mon, 22 Jan 2024 12:20:02 -0500 Subject: [PATCH 46/56] TSC meeting notes. (#1338) --- tsc/meetings/2024-01-17.md | 39 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100644 tsc/meetings/2024-01-17.md diff --git a/tsc/meetings/2024-01-17.md b/tsc/meetings/2024-01-17.md new file mode 100644 index 000000000..55e355faf --- /dev/null +++ b/tsc/meetings/2024-01-17.md @@ -0,0 +1,39 @@ +# OpenCue TSC Meeting Notes 17 Jan 2024 + +Secretary: Brian Cipriano + +Agenda/Notes: + +* Upgrade Cuebot to Java 21 + * Code changes in progress, PR coming. Let's discuss this next meeting once Diego is back, + interested to get more detail on any non-backwards-compatible changes. +* PR to fix crash in RQD with non-ascii chars + * Short term, fixing a crash is a priority. + * Longer term, there's a potential future project to improve non-ascii compatibility through the + system. +* Refactoring Cuebot host report handler + * PR coming soon. +* RQD changes for OOM protection + * PR coming soon. +* CueGUI web UI project launching at SPI + * Next.js on top of React. + * REST wrapper for gRPC. Many folks will find this useful outside of the GUI. + * Prototypes provide readonly functionality, for now. + * We should discuss the authorization system more, things are currently wide open if you have + network access. +* Blender plugin update + * Working on plugin bug fixes. + * Docs update coming soon. Packaging/distribution/install/upgrade process still a big open + question, we should look at the updated docs to see what the process currently looks like then + formulate a plan for this. + * RQD GPU image needs another look. +* M2/M3 compatibility + * We have been working on M1 compatibility for OpenCue. Are M2/M3 also supported? We're not + sure, let's check. + * Still need to finish the embedded postgres fix to complete M1 compatibility. +* OpenJobDescription + * https://github.com/OpenJobDescription/openjd-specifications/wiki + * Effort to standardize job descriptions between work management systems. + * OpenCue open to this effort? In theory yes, we will need to look into more detail to see what + would be required. Could add a layer for converting between the new job description and + OpenCue's internal format. From 9b4f90fd61df2f83d49e81a5e14388b67d0cc10f Mon Sep 17 00:00:00 2001 From: Diego Tavares da Silva Date: Mon, 29 Jan 2024 12:05:15 -0800 Subject: [PATCH 47/56] Fix GUI not updating bugs (#1180) * Fix GUI not updating bugs - Check if regex expression is empty before querying - Reduce timeout to a workable value. According to (gRPC Python docs)[https://grpc.github.io/grpc/python/grpc.html#grpc.UnaryUnaryMultiCallable.__call__], timeout is measured in seconds, meaning the current configuration is 2.7h which doesn't seem useful. Reducing to 10s. - Reducing JobMonitor tick to 10s - Check if regex expression is empty before querying - Fix JobMonitorTree getJobs call * Apply suggestions from code review Signed-off-by: Diego Tavares da Silva * Update MonitorJobsPlugin.py --------- Signed-off-by: Diego Tavares da Silva --- cuegui/cuegui/JobMonitorTree.py | 2 +- cuegui/cuegui/plugins/MonitorJobsPlugin.py | 20 ++++++++++---------- pycue/opencue/api.py | 1 + 3 files changed, 12 insertions(+), 11 deletions(-) diff --git a/cuegui/cuegui/JobMonitorTree.py b/cuegui/cuegui/JobMonitorTree.py index 7b829e959..5cb49f3d0 100644 --- a/cuegui/cuegui/JobMonitorTree.py +++ b/cuegui/cuegui/JobMonitorTree.py @@ -494,7 +494,7 @@ def _getUpdate(self): # an empty list for the id argument! if not ids: continue - tmp = opencue.api.getJobs(id=ids, all=True) + tmp = opencue.api.getJobs(id=ids, include_finished=True) self.__dependentJobs[job] = tmp if self.__loadMine: diff --git a/cuegui/cuegui/plugins/MonitorJobsPlugin.py b/cuegui/cuegui/plugins/MonitorJobsPlugin.py index a8ee4e034..875da70db 100644 --- a/cuegui/cuegui/plugins/MonitorJobsPlugin.py +++ b/cuegui/cuegui/plugins/MonitorJobsPlugin.py @@ -197,17 +197,17 @@ def _regexLoadJobsHandle(self): self.jobMonitor.removeAllItems() - if cuegui.Utils.isStringId(substring): - # If a uuid is provided, load it - self.jobMonitor.addJob(substring) - elif load_finished_jobs or re.search( + if substring: + if cuegui.Utils.isStringId(substring): + # If a uuid is provided, load it + self.jobMonitor.addJob(substring) + elif load_finished_jobs or re.search( r"^([a-z0-9_]+)\-([a-z0-9\.]+)\-", substring, re.IGNORECASE): - # If show and shot is provided, or if "load finished" checkbox is checked, load all jobs - for job in opencue.api.getJobs(regex=[substring], include_finished=True): - self.jobMonitor.addJob(job) - else: - # Otherwise, just load current matching jobs (except for the empty string) - if substring: + # Load all ff show and shot is provided or if "load finished" checkbox is checked + for job in opencue.api.getJobs(regex=[substring], include_finished=True): + self.jobMonitor.addJob(job) + else: + # Otherwise, just load current matching jobs (except for the empty string) for job in opencue.api.getJobs(regex=[substring]): self.jobMonitor.addJob(job) diff --git a/pycue/opencue/api.py b/pycue/opencue/api.py index 075d5c374..d43575496 100644 --- a/pycue/opencue/api.py +++ b/pycue/opencue/api.py @@ -325,6 +325,7 @@ def getJobs(**options): - show: show names - list - shot: shot names - list - user: user names - list + - include_finished - bool :rtype: list :return: a list of Job objects From 42542c1ad253e66a9c0f9dc81ba1d556883144ca Mon Sep 17 00:00:00 2001 From: Diego Tavares da Silva Date: Tue, 30 Jan 2024 09:44:07 -0800 Subject: [PATCH 48/56] Add new indexes to improve booking performance (#1304) * Add new indexes to improve booking performance After the changes on the gpu PR https://github.com/AcademySoftwareFoundation/OpenCue/pull/924 the performance of the booking query degraded up to 4 times the previous throughput. Creating some indexes for columns that changed names seems to have fixed the problem. Signed-off-by: Diego Tavares * Update cuebot/src/main/resources/conf/ddl/postgres/migrations/V18_Add_New_Indexes Signed-off-by: Diego Tavares da Silva * Update cuebot/src/main/resources/conf/ddl/postgres/migrations/V18_Add_New_Indexes Signed-off-by: Diego Tavares da Silva * Update cuebot/src/main/resources/conf/ddl/postgres/migrations/V18_Add_New_Indexes Signed-off-by: Diego Tavares da Silva --------- Signed-off-by: Diego Tavares Signed-off-by: Diego Tavares da Silva --- .../postgres/migrations/V18_Add_New_Indexes | 29 +++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 cuebot/src/main/resources/conf/ddl/postgres/migrations/V18_Add_New_Indexes diff --git a/cuebot/src/main/resources/conf/ddl/postgres/migrations/V18_Add_New_Indexes b/cuebot/src/main/resources/conf/ddl/postgres/migrations/V18_Add_New_Indexes new file mode 100644 index 000000000..107f426a0 --- /dev/null +++ b/cuebot/src/main/resources/conf/ddl/postgres/migrations/V18_Add_New_Indexes @@ -0,0 +1,29 @@ + +--Performance issue, Created new index on column int_gpus_min + + +CREATE INDEX IF NOT EXISTS i_layer_int_gpu_mem_min + ON public.layer USING btree + (int_gpus_min ASC NULLS LAST) + TABLESPACE pg_default; + + +CREATE INDEX IF NOT EXISTS i_layer_int_gpu_mem_min_1 + ON public.layer USING btree + (int_gpu_min ASC NULLS LAST) + TABLESPACE pg_default; + + +create index concurrently i_layer_int_cores_max on layer(int_cores_max); + +create index concurrently i_job_resource_int_priority on job_resource(int_priority); + +create index concurrently i_job_int_min_cores on job(int_min_cores); + +create index concurrently i_layer_limit_pk_layer on layer_limit(pk_layer); + +create index concurrently i_folder_resource_int_cores on folder_resource(int_cores); + +create index concurrently i_job_ts_updated on job(ts_updated); + +create index concurrently i_layer_str_tags on layer(str_tags); From c1dcad9e4f627fddc88ce69cce9d1c56900c4ee6 Mon Sep 17 00:00:00 2001 From: Diego Tavares da Silva Date: Tue, 30 Jan 2024 09:44:24 -0800 Subject: [PATCH 49/56] Logview threadpool (#861) * Update dispatchQuery to use min_cores Sorting jobs only by priority causes a situation where low priority jobs can get starved by a constant flow of high priority jobs. The new formula adds a modifier to the sorting rank to take into account the number of cores the job is requesting and also the number of days the job is waiting on the queue. Priorities numbers over 200 will mostly override the formula and work as a priority only based scheduling. sort = priority + (100 * (1 - (job.cores/job.int_min_cores))) + (age in days) Besides that, also take layer_int_cores_min into account when filtering folder_resourse limitations to avoid allocating more cores than the folder limits. (cherry picked from commit 566411aeeddc60983a30eabe121fd03263d05525) * Revert "Update dispatchQuery to use min_cores" This reverts commit 2eb4936c * Add threadpool logic for handling incoming log update requests (cherry picked from commit 351e210a9f0d49bfe61e9a57bba349bf1951af4e) * Suppress known ui warning messages For some ui process that especially involves multi threading or concurrent update of widget data, some warnings can happen due to obsolete function calls. The code has been updated to suppress a couple of known warnings (cherry picked from commit dd0c4f900fb75d25d63897b6aacd2d0b02e230f5) * Fix logview unit tests to work around the threading logic (cherry picked from commit 09446bbe5f0c631c74525cbc78b0e60ce114a2e2) * FIx unit tests * Fix python lint * Fix python lint * Fix lint Signed-off-by: Diego Tavares da Silva --------- Signed-off-by: Diego Tavares da Silva Co-authored-by: Fermi Perumal --- cuegui/cuegui/Main.py | 1 - cuegui/cuegui/plugins/LogViewPlugin.py | 141 ++++++++++++++------ cuegui/tests/plugins/LogViewPlugin_tests.py | 16 +-- 3 files changed, 105 insertions(+), 53 deletions(-) diff --git a/cuegui/cuegui/Main.py b/cuegui/cuegui/Main.py index 24f08864b..e46102b56 100644 --- a/cuegui/cuegui/Main.py +++ b/cuegui/cuegui/Main.py @@ -92,7 +92,6 @@ def startup(app_name, app_version, argv): app.aboutToQuit.connect(closingTime) # pylint: disable=no-member app.exec_() - def closingTime(): """Window close callback.""" logger.info("Closing all threads...") diff --git a/cuegui/cuegui/plugins/LogViewPlugin.py b/cuegui/cuegui/plugins/LogViewPlugin.py index 1e67ac1c4..5c36ec717 100644 --- a/cuegui/cuegui/plugins/LogViewPlugin.py +++ b/cuegui/cuegui/plugins/LogViewPlugin.py @@ -25,7 +25,9 @@ import os import re import string +import sys import time +import traceback from qtpy import QtGui from qtpy import QtCore @@ -292,10 +294,40 @@ def line_number_area_paint_event(self, event): bottom = top + self.blockBoundingRect(block).height() block_number += 1 +class LogLoadSignals(QtCore.QObject): + """Signals for the LoadLog action""" + SIG_LOG_LOAD_ERROR = QtCore.Signal(tuple) + SIG_LOG_LOAD_RESULT = QtCore.Signal(str, str) + SIG_LOG_LOAD_FINISHED = QtCore.Signal() + +class LogLoader(QtCore.QRunnable): + """A thread to load logs""" + def __init__(self, fn, *args, **kwargs): + super(LogLoader, self).__init__() + self.fn = fn + self.args = args + self.kwargs = kwargs + self.signals = LogLoadSignals() + + @QtCore.Slot() + def run(self): + # pylint: disable=bare-except + try: + content, log_mtime = self.fn(*self.args, **self.kwargs) + except: + exctype, value = sys.exc_info()[:2] + self.signals.SIG_LOG_LOAD_ERROR.emit( + (exctype, value, traceback.format_exc())) + else: + self.signals.SIG_LOG_LOAD_RESULT.emit(content, log_mtime) + finally: + self.signals.SIG_LOG_LOAD_FINISHED.emit() class LogViewWidget(QtWidgets.QWidget): - """Displays the log file for the selected frame.""" - + """ + Displays the log file for the selected frame + """ + SIG_CONTENT_UPDATED = QtCore.Signal(str, str) def __init__(self, parent=None): """ Create the UI elements @@ -453,6 +485,9 @@ def __init__(self, parent=None): self._current_match = 0 self._content_box.mousePressedSignal.connect(self._on_mouse_pressed) + self.SIG_CONTENT_UPDATED.connect(self._update_log_content) + self.log_thread_pool = QtCore.QThreadPool() + def _on_mouse_pressed(self, pos): """ Mouse press event, to be called when the user scrolls by hand or moves @@ -788,12 +823,56 @@ def _display_log_content(self): """ try: - self._update_log() - self._new_log = False + if not os.path.exists(self._log_file): + self._log_file_exists = False + content = 'Log file does not exist: %s' % self._log_file + self._content_timestamp = time.time() + self._update_log_content(content, self._log_mtime) + else: + # Creating the load logs process as qrunnables so + # that they don't block the ui while loading + log_loader = LogLoader(self._load_log, self._log_file, + self._new_log, self._log_mtime) + log_loader.signals.SIG_LOG_LOAD_RESULT.connect( + self._receive_log_results) + log_loader.setAutoDelete(True) + self.log_thread_pool.start(log_loader) + self.log_thread_pool.waitForDone() + self._new_log = False finally: QtCore.QTimer.singleShot(5000, self._display_log_content) - def _update_log(self): + # pylint: disable=no-self-use + @QtCore.Slot() + def _load_log(self, log_file, new_log, curr_log_mtime): + content = None + log_size = int(os.stat(log_file).st_size) + if log_size > 1 * 1e6: + content = ('Log file size (%0.1f MB) exceeds the size ' + 'threshold (1.0 MB).' + % float(log_size / (1024 * 1024))) + elif not new_log and os.path.exists(log_file): + log_mtime = os.path.getmtime(log_file) + if log_mtime > curr_log_mtime: + curr_log_mtime = log_mtime # no new updates + content = '' + + if content is None: + content = '' + try: + with open(log_file, 'r') as f: + content = f.read() + except IOError: + content = 'Can not access log file: %s' % log_file + + return content, curr_log_mtime + + @QtCore.Slot() + def _receive_log_results(self, content, log_mtime): + self.SIG_CONTENT_UPDATED.emit(content, log_mtime) + + @QtCore.Slot(str, str) + def _update_log_content(self, content, log_mtime): """ Updates the content of the content box with the content of the log file, if necessary. The full path to the log file will be populated in @@ -813,49 +892,23 @@ def _update_log(self): (if necessary) """ - # Get the content of the log file - if not self._log_file: - return # There's no log file, nothing to do here! - self._path.setText(self._log_file) - content = None - if not os.path.exists(self._log_file): - self._log_file_exists = False - content = 'Log file does not exist: %s' % self._log_file - self._content_timestamp = time.time() - else: - log_size = int(os.stat(self._log_file).st_size) - if log_size > 5 * 1e6: - content = ('Log file size (%0.1f MB) exceeds the size ' - 'threshold (5.0 MB).' - % float(log_size / (1024 * 1024))) - elif not self._new_log and os.path.exists(self._log_file): - log_mtime = os.path.getmtime(self._log_file) - if log_mtime > self._log_mtime: - self._log_mtime = log_mtime # no new updates - content = '' - - if content is None: - content = '' - try: - with open(self._log_file, 'r') as f: - content = f.read() - except IOError: - content = 'Can not access log file: %s' % self._log_file + self._log_mtime = log_mtime - # Do we need to scroll to the end? - scroll_to_end = (self._scrollbar_max == self._scrollbar_value - or self._new_log) + self.app.processEvents() # Update the content in the gui (if necessary) - current_text = (self._content_box.toPlainText() or '') - new_text = content.lstrip(str(current_text)) - if new_text: - if self._new_log: - self._content_box.setPlainText(content) - else: + if self._new_log: + self._content_box.setPlainText(content) + else: + current_text = (self._content_box.toPlainText() or '') + new_text = content.lstrip(str(current_text)) + if new_text: self._content_box.appendPlainText(new_text) - self._content_timestamp = time.time() - self.app.processEvents() + self._content_timestamp = time.time() + self._path.setText(self._log_file) + + scroll_to_end = (self._scrollbar_max == self._scrollbar_value + or self._new_log) # Adjust scrollbar value (if necessary) self._scrollbar_max = self._log_scrollbar.maximum() diff --git a/cuegui/tests/plugins/LogViewPlugin_tests.py b/cuegui/tests/plugins/LogViewPlugin_tests.py index a135747b5..62752c2f3 100644 --- a/cuegui/tests/plugins/LogViewPlugin_tests.py +++ b/cuegui/tests/plugins/LogViewPlugin_tests.py @@ -71,21 +71,22 @@ def setUp(self): def test_shouldDisplayFirstLogFile(self): cuegui.app().display_log_file_content.emit([self.logPath1, self.logPath2]) - + self.logViewPlugin.logview_widget._receive_log_results(_LOG_TEXT_1, 0) self.assertEqual(_LOG_TEXT_1, self.logViewPlugin.logview_widget._content_box.toPlainText()) def test_shouldUpdateLogFile(self): cuegui.app().display_log_file_content.emit([self.logPath1, self.logPath2]) + self.logViewPlugin.logview_widget._receive_log_results(_LOG_TEXT_1, 0) new_contents = _LOG_TEXT_1 + '\nanother line at the end' self.log1.set_contents(new_contents) cuegui.app().display_log_file_content.emit([self.logPath1, self.logPath2]) - + self.logViewPlugin.logview_widget._receive_log_results(new_contents, 0) self.assertEqual(new_contents, self.logViewPlugin.logview_widget._content_box.toPlainText()) def test_shouldHighlightAllSearchResults(self): - cuegui.app().display_log_file_content.emit([self.logPath1, self.logPath2]) + self.logViewPlugin.logview_widget._receive_log_results(_LOG_TEXT_1, 0) self.logViewPlugin.logview_widget._case_stv_checkbox.setCheckState( - qtpy.QtCore.Qt.CheckState.Unchecked) + qtpy.QtCore.Qt.CheckState.Unchecked) self.logViewPlugin.logview_widget._search_box.setText('lorem') self.logViewPlugin.logview_widget._search_button.click() @@ -100,7 +101,7 @@ def test_shouldHighlightAllSearchResults(self): self.logViewPlugin.logview_widget._content_box, matches[1][0], matches[1][1])) def test_shouldMoveCursorToSecondSearchResult(self): - cuegui.app().display_log_file_content.emit([self.logPath1, self.logPath2]) + self.logViewPlugin.logview_widget._receive_log_results(_LOG_TEXT_1, 0) self.logViewPlugin.logview_widget._case_stv_checkbox.setCheckState( qtpy.QtCore.Qt.CheckState.Unchecked) @@ -114,7 +115,7 @@ def test_shouldMoveCursorToSecondSearchResult(self): self.assertEqual(132, self.logViewPlugin.logview_widget._cursor.position()) def test_shouldMoveCursorLastSearchResult(self): - cuegui.app().display_log_file_content.emit([self.logPath1, self.logPath2]) + self.logViewPlugin.logview_widget._receive_log_results(_LOG_TEXT_1, 0) self.logViewPlugin.logview_widget._case_stv_checkbox.setCheckState( qtpy.QtCore.Qt.CheckState.Unchecked) @@ -128,10 +129,9 @@ def test_shouldMoveCursorLastSearchResult(self): self.assertEqual(132, self.logViewPlugin.logview_widget._cursor.position()) def test_shouldPerformCaseInsensitiveSearch(self): - cuegui.app().display_log_file_content.emit([self.logPath1, self.logPath2]) + self.logViewPlugin.logview_widget._receive_log_results(_LOG_TEXT_1, 0) self.logViewPlugin.logview_widget._case_stv_checkbox.setCheckState( qtpy.QtCore.Qt.CheckState.Checked) - self.logViewPlugin.logview_widget._search_box.setText('lorem') self.logViewPlugin.logview_widget._search_button.click() matches = self.logViewPlugin.logview_widget._matches From a694b8dce53e6adf804abfe6a758316946e87124 Mon Sep 17 00:00:00 2001 From: Angela Li <144162000+angelali-ms@users.noreply.github.com> Date: Wed, 7 Feb 2024 08:17:44 -0800 Subject: [PATCH 50/56] [rqd] Add SYSTEMDRIVE to env var list for Windows machines (#1341) --- rqd/rqd/rqcore.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rqd/rqd/rqcore.py b/rqd/rqd/rqcore.py index 842028732..9391a2126 100644 --- a/rqd/rqd/rqcore.py +++ b/rqd/rqd/rqcore.py @@ -101,7 +101,7 @@ def __createEnvVariables(self): self.frameEnv["MAIL"] = "/usr/mail/%s" % self.runFrame.user_name self.frameEnv["HOME"] = "/net/homedirs/%s" % self.runFrame.user_name elif platform.system() == "Windows": - for variable in ["SYSTEMROOT", "APPDATA", "TMP", "COMMONPROGRAMFILES"]: + for variable in ["SYSTEMROOT", "APPDATA", "TMP", "COMMONPROGRAMFILES", "SYSTEMDRIVE"]: if variable in os.environ: self.frameEnv[variable] = os.environ[variable] From 2ddd3dbba41cce13dff0bc329970bc08438c5d18 Mon Sep 17 00:00:00 2001 From: Brian Cipriano Date: Thu, 15 Feb 2024 15:56:49 -0500 Subject: [PATCH 51/56] Notes from TSC meeting. (#1344) --- tsc/meetings/2024-02-14.md | 39 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100644 tsc/meetings/2024-02-14.md diff --git a/tsc/meetings/2024-02-14.md b/tsc/meetings/2024-02-14.md new file mode 100644 index 000000000..2317d1396 --- /dev/null +++ b/tsc/meetings/2024-02-14.md @@ -0,0 +1,39 @@ +# OpenCue TSC Meeting Notes 14 Feb 2024 + +Secretary: Brian Cipriano + +Agenda/Notes: + +* Cuebot upgrade to Java 21 + * Started to refactor HostReportHandler, realized it would be much improved using newer java + features such as thread pools. + * Upgraded java and some dependencies like gradle. + * SPI has a build that passes all unit tests and are currently testing on a dev environment + before moving forward. Will do some a/b performance testing as well. + * We'll need to upgrade java on our CI docker images, this might require a different base image. + * Can discuss more on github issue/PR. +* OpenJobDescription + * Review of their project goals as discussed last time. Code published to their github org now. + * Reviewed the different github repos and their purpose. + * SPI to review further. + * OpenJD appears to be a superset of opencue's job schema, so opencue should fit into this + system fine. + * Could do implementation in different stages, start with a simple openjd -> opencue converter, + later add other components such as the CLI into cuebot/RQD. + * Diego to start a thread on their discussion forum. + * Others to look into new github repos and understand how opencue would implement support. + * Longer term project. +* Web UI update + * Continued progress on prototyping. + * Will this be a full replacement, or desktop GUI app kept for power users? This is an open + question, starting with basic functionality and will see how it goes. +* RQD OOM issue + * Still testing, PR coming soon. +* Blender plugin + * Almost ready to merge PR with cuesubmit frame range adjustments, then will incorporate that + into the plugin. + * Draft of user guide google doc is ready, linked in the PR. Brian to review. +* opencue.io appears dead to casual users + * Not much new on there, new activity mostly limited to the github, no new releases recently. + * We should do regular website updates, monthly? Publish an activity report? + * We should publish a new release soon as well, been a while. From 3ce95a9367cca12153406c807243a31f26545e88 Mon Sep 17 00:00:00 2001 From: Ramon Figueiredo Date: Fri, 16 Feb 2024 07:54:52 -0800 Subject: [PATCH 52/56] [cuebot] Fix duplicate join in find_jobs_by_show. (#1343) --- .../java/com/imageworks/spcue/dao/postgres/DispatchQuery.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/DispatchQuery.java b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/DispatchQuery.java index 1d1f7210f..a789307af 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/DispatchQuery.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/DispatchQuery.java @@ -44,7 +44,6 @@ public class DispatchQuery { "AND folder.pk_dept = point.pk_dept " + "AND folder.pk_show = point.pk_show " + "AND job.pk_job = layer.pk_job " + - "AND job_resource.pk_job = job.pk_job " + "AND (CASE WHEN layer_stat.int_waiting_count > 0 THEN layer_stat.pk_layer ELSE NULL END) = layer.pk_layer " + "AND " + "(" + @@ -139,7 +138,6 @@ public class DispatchQuery { "AND folder.pk_dept = point.pk_dept " + "AND folder.pk_show = point.pk_show " + "AND job.pk_job = layer.pk_job " + - "AND job_resource.pk_job = job.pk_job " + "AND (CASE WHEN layer_stat.int_waiting_count > 0 THEN layer_stat.pk_layer ELSE NULL END) = layer.pk_layer " + "AND " + "(" + From 0d3cc6900af64dab98f63aa8dd64deb88644df50 Mon Sep 17 00:00:00 2001 From: Diego Tavares Date: Tue, 5 Mar 2024 09:35:34 -0800 Subject: [PATCH 53/56] Fix service override. (#866) --- .../spcue/servant/ManageServiceOverride.java | 13 +++++---- cuegui/cuegui/ServiceDialog.py | 22 +++++++++------ pycue/opencue/cuebot.py | 1 + pycue/opencue/wrappers/service.py | 23 +++++++++++++++ pycue/opencue/wrappers/show.py | 28 +++++++++---------- 5 files changed, 60 insertions(+), 27 deletions(-) diff --git a/cuebot/src/main/java/com/imageworks/spcue/servant/ManageServiceOverride.java b/cuebot/src/main/java/com/imageworks/spcue/servant/ManageServiceOverride.java index 16afb3bf2..6d2db02fe 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/servant/ManageServiceOverride.java +++ b/cuebot/src/main/java/com/imageworks/spcue/servant/ManageServiceOverride.java @@ -23,7 +23,7 @@ import io.grpc.stub.StreamObserver; -import com.imageworks.spcue.ServiceEntity; +import com.imageworks.spcue.ServiceOverrideEntity; import com.imageworks.spcue.grpc.service.Service; import com.imageworks.spcue.grpc.service.ServiceOverrideDeleteRequest; import com.imageworks.spcue.grpc.service.ServiceOverrideDeleteResponse; @@ -39,7 +39,8 @@ public class ManageServiceOverride extends ServiceOverrideInterfaceGrpc.ServiceO @Override public void delete(ServiceOverrideDeleteRequest request, StreamObserver responseObserver) { - serviceManager.deleteService(toServiceEntity(request.getService())); + // Passing null on showId as the interface doesn't require a showId in this situation + serviceManager.deleteService(toServiceOverrideEntity(request.getService(), null)); responseObserver.onNext(ServiceOverrideDeleteResponse.newBuilder().build()); responseObserver.onCompleted(); } @@ -47,7 +48,8 @@ public void delete(ServiceOverrideDeleteRequest request, @Override public void update(ServiceOverrideUpdateRequest request, StreamObserver responseObserver) { - serviceManager.updateService(toServiceEntity(request.getService())); + // Passing null on showId as the interface doesn't require a showId in this situation + serviceManager.updateService(toServiceOverrideEntity(request.getService(), null)); responseObserver.onNext(ServiceOverrideUpdateResponse.newBuilder().build()); responseObserver.onCompleted(); } @@ -60,8 +62,8 @@ public void setServiceManager(ServiceManager serviceManager) { this.serviceManager = serviceManager; } - private ServiceEntity toServiceEntity(Service service) { - ServiceEntity entity = new ServiceEntity(); + private ServiceOverrideEntity toServiceOverrideEntity(Service service, String showId){ + ServiceOverrideEntity entity = new ServiceOverrideEntity(); entity.id = service.getId(); entity.name = service.getName(); entity.minCores = service.getMinCores(); @@ -72,6 +74,7 @@ private ServiceEntity toServiceEntity(Service service) { entity.minGpuMemory = service.getMinGpuMemory(); entity.tags = new LinkedHashSet<>(service.getTagsList()); entity.threadable = service.getThreadable(); + entity.showId = showId; entity.timeout = service.getTimeout(); entity.timeout_llu = service.getTimeoutLlu(); entity.minMemoryIncrease = service.getMinMemoryIncrease(); diff --git a/cuegui/cuegui/ServiceDialog.py b/cuegui/cuegui/ServiceDialog.py index c54ca4a88..1fac432af 100644 --- a/cuegui/cuegui/ServiceDialog.py +++ b/cuegui/cuegui/ServiceDialog.py @@ -31,6 +31,7 @@ import cuegui.Constants import cuegui.TagsWidget import cuegui.Utils +from opencue.wrappers.service import ServiceOverride class ServiceForm(QtWidgets.QWidget): @@ -124,13 +125,13 @@ def setService(self, service): """ self.__service = service self.__buttons.setDisabled(False) - self.name.setText(service.data.name) - self.threadable.setChecked(service.data.threadable) - self.min_cores.setValue(service.data.min_cores) - self.max_cores.setValue(service.data.max_cores) - self.min_memory.setValue(service.data.min_memory // 1024) + self.name.setText(service.name()) + self.threadable.setChecked(service.threadable()) + self.min_cores.setValue(service.minCores()) + self.max_cores.setValue(service.maxCores()) self.min_gpu_memory.setValue(service.data.min_gpu_memory // 1024) - self._tags_w.set_tags(service.data.tags) + self.min_memory.setValue(service.minMemory() // 1024) + self._tags_w.set_tags(service.tags()) self.timeout.setValue(service.data.timeout) self.timeout_llu.setValue(service.data.timeout_llu) self.min_memory_increase.setValue(service.data.min_memory_increase // 1024) @@ -263,11 +264,16 @@ def saved(self, service): if self.__new_service: if self.__show: - self.__show.createServiceOverride(service.data) + serviceOverride = self.__show.createServiceOverride(service.data) else: opencue.api.createService(service.data) else: - service.update() + if self.__show: + serviceOverride = ServiceOverride(service) + serviceOverride.id = service.id() + serviceOverride.update() + else: + service.update() self.refresh() self.__new_service = False diff --git a/pycue/opencue/cuebot.py b/pycue/opencue/cuebot.py index b007f0efd..764d5428e 100644 --- a/pycue/opencue/cuebot.py +++ b/pycue/opencue/cuebot.py @@ -136,6 +136,7 @@ class Cuebot(object): 'proc': host_pb2_grpc.ProcInterfaceStub, 'renderPartition': renderPartition_pb2_grpc.RenderPartitionInterfaceStub, 'service': service_pb2_grpc.ServiceInterfaceStub, + 'serviceOverride': service_pb2_grpc.ServiceOverrideInterfaceStub, 'show': show_pb2_grpc.ShowInterfaceStub, 'subscription': subscription_pb2_grpc.SubscriptionInterfaceStub, 'task': task_pb2_grpc.TaskInterfaceStub diff --git a/pycue/opencue/wrappers/service.py b/pycue/opencue/wrappers/service.py index b3f6563d2..92cafe7fc 100644 --- a/pycue/opencue/wrappers/service.py +++ b/pycue/opencue/wrappers/service.py @@ -259,3 +259,26 @@ def setMinMemoryIncrease(self, min_memory_increase): self.data.min_memory_increase = min_memory_increase else: raise ValueError("Minimum memory increase must be > 0") + +class ServiceOverride(object): + def __init__(self, serviceOverride=None): + if serviceOverride: + self.id = serviceOverride.id + self.data = serviceOverride.data or service_pb2.Service().data + else: + defaultServiceOverride = service_pb2.ServiceOverride() + self.id = defaultServiceOverride.id + self.data = defaultServiceOverride.data + + self.stub = Cuebot.getStub("serviceOverride") + + def delete(self): + self.stub.Delete( + service_pb2.ServiceOverrideDeleteRequest(service=self.data), + timeout=Cuebot.Timeout) + + def update(self): + """Commit a ServiceOverride change to the database""" + self.stub.Update( + service_pb2.ServiceOverrideUpdateRequest(service=self.data), + timeout=Cuebot.Timeout) diff --git a/pycue/opencue/wrappers/show.py b/pycue/opencue/wrappers/show.py index 5dad1a000..c7594aaa3 100644 --- a/pycue/opencue/wrappers/show.py +++ b/pycue/opencue/wrappers/show.py @@ -19,6 +19,7 @@ import opencue.wrappers.filter import opencue.wrappers.group import opencue.wrappers.subscription +from opencue.wrappers.service import ServiceOverride class Show(object): @@ -66,30 +67,29 @@ def delete(self): def createServiceOverride(self, data): """Creates a Service Override at the show level. - - :type data: service_pb2.Service - :param data: service data, typically from opencue.wrappers.service.Service.data + :type data: opencue.wrapper.service.Service + :param data: Service.data object """ # min_memory_increase has to be greater than 0. if data.min_memory_increase <= 0: raise ValueError("Minimum memory increase must be > 0") - - self.stub.CreateServiceOverride( - show_pb2.ShowCreateServiceOverrideRequest(show=self.data, service=data), - timeout=Cuebot.Timeout) + + self.stub.CreateServiceOverride(show_pb2.ShowCreateServiceOverrideRequest( + show=self.data, service=data), + timeout=Cuebot.Timeout) def getServiceOverride(self, serviceName): - """Returns a service override for a show. + """ + Returns a service override for a show - :type serviceName: str :param serviceName: name of the service for the show - :rtype: service_pb2.ServiceOverride :return: service override object """ - return self.stub.GetServiceOverride( - show_pb2.ShowGetServiceOverrideRequest(show=self.data, name=serviceName), - timeout=Cuebot.Timeout).service_override + serviceOverride = self.stub.GetServiceOverride(show_pb2.ShowGetServiceOverrideRequest( + show=self.data, name=serviceName), + timeout=Cuebot.Timeout).service_override + return ServiceOverride(serviceOverride) def getServiceOverrides(self): """Returns a list of service overrides on the show. @@ -100,7 +100,7 @@ def getServiceOverrides(self): serviceOverrideSeq = self.stub.GetServiceOverrides( show_pb2.ShowGetServiceOverridesRequest(show=self.data), timeout=Cuebot.Timeout).service_overrides - return serviceOverrideSeq.service_overrides + return [ServiceOverride(override) for override in serviceOverrideSeq.service_overrides] def getSubscriptions(self): """Returns a list of all subscriptions the show has. From b140fa63ed6c53d0587172a0e186c4a21ebfabba Mon Sep 17 00:00:00 2001 From: Brian Cipriano Date: Thu, 4 Apr 2024 13:32:36 -0400 Subject: [PATCH 54/56] TSC meeting notes. (#1352) --- tsc/meetings/2024-03-27.md | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100644 tsc/meetings/2024-03-27.md diff --git a/tsc/meetings/2024-03-27.md b/tsc/meetings/2024-03-27.md new file mode 100644 index 000000000..e2cb77227 --- /dev/null +++ b/tsc/meetings/2024-03-27.md @@ -0,0 +1,26 @@ +# OpenCue TSC Meeting Notes 27 Mar 2024 + +Secretary: Brian Cipriano + +Agenda/Notes: + +* TSC handoff + * Brian has been TSC chair for 5-6 years. At this point his time is limited due to other work + priorities. TSC and ASWF participation has been suffering as a result. + * It's a good idea to rotate chairs regularly anyway for project health. + * Will work offline to discuss among TSC members and find a new chair. + * Brian will stay around to participate in the project, and will be heavily involved in the + handoff to make sure the new chair is comfortable with new duties. +* New web UI version + * Prototype coming in early may. + * Still planned as readonly. + * Authentication: + * okta within SPI + * By default it will need to use the database, need to create User as first-class object. + * Login with X (google / github / etc.) in the future? + * Brian: rolling our own auth system feels very old school. There must be a better way. +* New Nimby notifications on desktop + * Disabled by default, new constant to enable. + * Tkinter for showing notifications, so it's cross-platform. +* Java 21 upgrade + * Ongoing, most tests complete, PR still being prepared. From 3e2a93218f235a19a60108653561499780ef5cf9 Mon Sep 17 00:00:00 2001 From: Brian Cipriano Date: Wed, 24 Apr 2024 18:16:59 -0400 Subject: [PATCH 55/56] TSC meeting notes. (#1357) --- tsc/meetings/2024-04-24.md | 41 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) create mode 100644 tsc/meetings/2024-04-24.md diff --git a/tsc/meetings/2024-04-24.md b/tsc/meetings/2024-04-24.md new file mode 100644 index 000000000..8a0a20091 --- /dev/null +++ b/tsc/meetings/2024-04-24.md @@ -0,0 +1,41 @@ +# OpenCue TSC Meeting Notes 24 Apr 2024 + +Secretary: Brian Cipriano + +Agenda/Notes: + +* TSC chair handoff + * Diego has volunteered to take over TSC chair position + * No objections from the TSC, new chair position accepted! + * Related housekeeping: + * Meetings + * Need to migrate to new ASWF scheduling system + * Also need to move to Zoom, Brian owns current GVC meeting invite + * Brian to reach out to John Mertic about this + * Consider starting new agenda / notes doc, whatever works best with the new meeting system + * Good chance to clear out Github CODEOWNERS / email lists / groups + * Let's add Ramon to CODEOWNERS + * Can we identify new reviewers / committers automatically with a report of top + contributors? +* Web version + * REST gateway: https://github.com/AcademySoftwareFoundation/OpenCue/pull/1355 + * CueWeb: https://github.com/AcademySoftwareFoundation/OpenCue/pull/1356 + * Currently going to separate branch. + * Security issues + * Currently readonly, but write functionality will be added over time + * Very dangerous if deployed to the public internet + * There are no plans to do this currently, but once code is committed someone could do this. + It's a potential liability issue for OpenCue. + * Let's add additional warnings and keep all code on the separate branch for now. + * Web UI uses a separate REST gateway, which is even more useful than the Web UI. Has the same + security concerns. Perimeter security is not reliable — phishing emails, etc. + * Let's review the two PRs for initial thoughts, testing continues within SPI. +* Daniel: OpenJobDescription + * Potential summer project to implement some initial OpenCue integration. + * Early days, still not decided whether we will get a slot allotted to us. + * Diego offered to help onboard to the OpenCue codebase if we get to that. +* Nuwan: Blender update + * Brian to keep working with Nuwan on this, even after TSC chair handoff. + * Google doc with setup instructions needs another review. + * PR with some opencue.io changes also needs a review. + * Looking at Kubernetes integration for a potential next project. From b0dff61c42251eee26c49799958d8bd76562e4c7 Mon Sep 17 00:00:00 2001 From: Akim Ruslanov Date: Fri, 3 May 2024 14:01:04 -0700 Subject: [PATCH 56/56] Raise error if searching with incorrect parameter (#1165) --- pycue/opencue/search.py | 4 ++++ pycue/tests/api_test.py | 21 ++++++++++++++++++++- 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/pycue/opencue/search.py b/pycue/opencue/search.py index 1a42f696a..80ad0fa5e 100644 --- a/pycue/opencue/search.py +++ b/pycue/opencue/search.py @@ -381,4 +381,8 @@ def _setOptions(criteria, options): criteria.first_result = int(v) elif k == "include_finished": criteria.include_finished = v + elif len(k) == 0: + return criteria + else: + raise Exception("Criteria for search does not exist") return criteria diff --git a/pycue/tests/api_test.py b/pycue/tests/api_test.py index 759c09da1..b75fcec3f 100644 --- a/pycue/tests/api_test.py +++ b/pycue/tests/api_test.py @@ -189,7 +189,7 @@ def testGetJobs(self, getStubMock): jobs=job_pb2.JobSeq(jobs=[job_pb2.Job(name=TEST_JOB_NAME)])) getStubMock.return_value = stubMock - jobsByShow = opencue.api.getJobs(show=[TEST_SHOW_NAME], all=True) + jobsByShow = opencue.api.getJobs(show=[TEST_SHOW_NAME]) stubMock.GetJobs.assert_called_with( job_pb2.JobGetJobsRequest( @@ -206,6 +206,25 @@ def testGetJobs(self, getStubMock): self.assertEqual(1, len(jobsByName)) self.assertEqual(TEST_JOB_NAME, jobsByName[0].name()) + @mock.patch('opencue.cuebot.Cuebot.getStub') + def testGetAllJobs(self, getStubMock): + stubMock = mock.Mock() + stubMock.GetJobs.return_value = job_pb2.JobGetJobsResponse( + jobs=job_pb2.JobSeq(jobs=[job_pb2.Job(name=TEST_JOB_NAME)])) + getStubMock.return_value = stubMock + + jobs = opencue.api.getJobs() + + stubMock.GetJobs.assert_called_with( + job_pb2.JobGetJobsRequest( + r=job_pb2.JobSearchCriteria()), timeout=mock.ANY) + self.assertEqual(1, len(jobs)) + self.assertEqual(TEST_JOB_NAME, jobs[0].name()) + + def testRaiseExceptionOnBadCriteriaSearch(self): + with self.assertRaises(Exception) as context: + opencue.api.getJobs(bad_criteria=["00000000-0000-0000-0000-012345678980"]) + @mock.patch('opencue.cuebot.Cuebot.getStub') def testGetJob(self, getStubMock): arbitraryId = '00000000-0000-0000-0000-012345678980'