Merge branch 'AcademySoftwareFoundation:master' into cuesubmit-jobs-f…

…rom-config-file

.github/workflows/testing-pipeline.yml

@@ -7,90 +7,57 @@ on:
     branches: [ master ]
 
 jobs:
-  test_python_2019:
-    name: Run Python Unit Tests (CY2019)
-    runs-on: ubuntu-latest
-    container: aswf/ci-opencue:2019
-    steps:
-      - uses: actions/checkout@v3
-      - name: Run Python Tests
-        run: ci/run_python_tests.sh
-
-  test_cuebot_2019:
-    name: Build Cuebot and Run Unit Tests (CY2019)
-    runs-on: ubuntu-latest
-    container:
-      image: aswf/ci-opencue:2019
-    steps:
-    - uses: actions/checkout@v3
-    - name: Build with Gradle
-      run: |
-        chown -R aswfuser:aswfgroup .
-        su -c "cd cuebot && ./gradlew build --stacktrace --info" aswfuser
-
-  test_python_2020:
-    name: Run Python Unit Tests (CY2020)
+  test_python_2022:
+    name: Run Python Unit Tests (CY2022)
     runs-on: ubuntu-latest
-    container: aswf/ci-opencue:2020
+    container: aswf/ci-opencue:2022
     steps:
       - uses: actions/checkout@v3
       - name: Run Python Tests
-        run: ci/run_python_tests.sh
+        run: ci/run_python_tests.sh --no-gui
 
-  test_cuebot_2020:
-    name: Build Cuebot and Run Unit Tests (CY2020)
+  test_cuebot_2022:
+    name: Build Cuebot and Run Unit Tests (CY2022)
     runs-on: ubuntu-latest
     container:
-      image: aswf/ci-opencue:2020
+      image: aswf/ci-opencue:2022
     steps:
       - uses: actions/checkout@v3
       - name: Build with Gradle
         run: |
           chown -R aswfuser:aswfgroup .
           su -c "cd cuebot && ./gradlew build --stacktrace --info" aswfuser
 
-  test_python_2021:
-    name: Run Python Unit Tests (CY2021)
+  test_python_2023:
+    name: Run Python Unit Tests (CY2023)
     runs-on: ubuntu-latest
-    container: aswf/ci-opencue:2021
+    container: aswf/ci-opencue:2023
     steps:
       - uses: actions/checkout@v3
       - name: Run Python Tests
         run: ci/run_python_tests.sh
 
-  test_cuebot_2021:
-    name: Build Cuebot and Run Unit Tests (CY2021)
+  test_cuebot_2023:
+    name: Build Cuebot and Run Unit Tests (CY2023)
     runs-on: ubuntu-latest
     container:
-      image: aswf/ci-opencue:2021
+      image: aswf/ci-opencue:2023
     steps:
       - uses: actions/checkout@v3
       - name: Build with Gradle
         run: |
           chown -R aswfuser:aswfgroup .
           su -c "cd cuebot && ./gradlew build --stacktrace --info" aswfuser
 
-  test_python_2022:
-    name: Run Python Unit Tests (CY2022)
+  test_python2:
+    name: Run Python Unit Tests using Python2
     runs-on: ubuntu-latest
-    container: aswf/ci-opencue:2022
+    container: aswf/ci-opencue:2019
     steps:
       - uses: actions/checkout@v3
       - name: Run Python Tests
         run: ci/run_python_tests.sh
 
-  test_cuebot_2022:
-    name: Build Cuebot and Run Unit Tests (CY2022)
-    runs-on: ubuntu-latest
-    container:
-      image: aswf/ci-opencue:2022
-    steps:
-      - uses: actions/checkout@v3
-      - name: Build with Gradle
-        run: |
-          chown -R aswfuser:aswfgroup .
-          su -c "cd cuebot && ./gradlew build --stacktrace --info" aswfuser
-
   test_pyside6:
     name: Run CueGUI Tests using PySide6
     runs-on: ubuntu-latest
@@ -113,7 +80,7 @@ jobs:
     name: Test Documentation Build
     runs-on: ubuntu-latest
     container:
-      image: aswf/ci-opencue:2020
+      image: aswf/ci-opencue:2023
     steps:
       - uses: actions/checkout@v3
       - name: Run Sphinx build

ci/run_python_tests.sh

@@ -1,7 +1,13 @@
 #!/bin/bash
 
+# Script for running OpenCue unit tests with PySide2.
+#
+# This script is written to be run within the OpenCue GitHub Actions environment.
+# See `.github/workflows/testing-pipeline.yml`.
+
 set -e
 
+args=("$@")
 python_version=$(python -V 2>&1)
 echo "Will run tests using ${python_version}"
 
@@ -23,6 +29,6 @@ PYTHONPATH=pycue:pyoutline python cuesubmit/setup.py test
 python rqd/setup.py test
 
 # Xvfb no longer supports Python 2.
-if [[ "$python_version" =~ "Python 3" ]]; then
+if [[ "$python_version" =~ "Python 3" && ${args[0]} != "--no-gui" ]]; then
   ci/run_gui_test.sh
 fi

ci/run_python_tests_pyside6.sh

@@ -0,0 +1,48 @@
+#!/bin/bash
+
+# Script for running OpenCue unit tests with PySide6.
+#
+# This script is written to be run within the OpenCue GitHub Actions environment.
+# See `.github/workflows/testing-pipeline.yml`.
+
+set -e
+
+python_version=$(python -V 2>&1)
+echo "Will run tests using ${python_version}"
+
+# NOTE: To run this in an almalinux environment, install these packages:
+# yum -y install \
+#   dbus-libs \
+#   fontconfig \
+#   gcc \
+#   libxkbcommon-x11 \
+#   mesa-libEGL-devel \
+#   python-devel \
+#   which \
+#   xcb-util-keysyms \
+#   xcb-util-image \
+#   xcb-util-renderutil \
+#   xcb-util-wm \
+#   Xvfb
+
+# Install Python requirements.
+python3 -m pip install --user -r requirements.txt -r requirements_gui.txt
+# Replace PySide2 with PySide6.
+python3 -m pip uninstall -y PySide2
+python3 -m pip install --user PySide6==6.3.2
+
+# Protos need to have their Python code generated in order for tests to pass.
+python -m grpc_tools.protoc -I=proto/ --python_out=pycue/opencue/compiled_proto --grpc_python_out=pycue/opencue/compiled_proto proto/*.proto
+python -m grpc_tools.protoc -I=proto/ --python_out=rqd/rqd/compiled_proto --grpc_python_out=rqd/rqd/compiled_proto proto/*.proto
+
+# Fix compiled proto code for Python 3.
+2to3 -wn -f import pycue/opencue/compiled_proto/*_pb2*.py
+2to3 -wn -f import rqd/rqd/compiled_proto/*_pb2*.py
+
+python pycue/setup.py test
+PYTHONPATH=pycue python pyoutline/setup.py test
+PYTHONPATH=pycue python cueadmin/setup.py test
+PYTHONPATH=pycue:pyoutline python cuesubmit/setup.py test
+python rqd/setup.py test
+
+ci/run_gui_test.sh

cuebot/src/main/java/com/imageworks/spcue/dao/postgres/JobDaoJdbc.java

@@ -574,7 +574,7 @@ public void activateJob(JobInterface job, JobState jobState) {
                 jobTotals[0] + jobTotals[1], layers.size(), job.getJobId());
 
         getJdbcTemplate().update(
-                "UPDATE show SET int_frame_insert_count=int_frame_insert_count+?, int_job_insert_count=int_job_insert_count+1 WHERE pk_show=?",
+                "UPDATE show_stats SET int_frame_insert_count=int_frame_insert_count+?, int_job_insert_count=int_job_insert_count+1 WHERE pk_show=?",
                 jobTotals[0] + jobTotals[1], job.getShowId());
 
         updateState(job, jobState);

cuebot/src/main/java/com/imageworks/spcue/dao/postgres/ShowDaoJdbc.java

@@ -138,9 +138,15 @@ public ShowEntity getShowDetail(HostInterface host) {
     private static final String INSERT_SHOW =
         "INSERT INTO show (pk_show,str_name) VALUES (?,?)";
 
+    private static final String INSERT_SHOW_STATS =
+        "INSERT INTO show_stats " +
+            "(pk_show, int_frame_insert_count, int_job_insert_count, int_frame_success_count, int_frame_fail_count) " +
+            "VALUES (?, 0, 0, 0, 0)";
+
     public void insertShow(ShowEntity show) {
         show.id = SqlUtil.genKeyRandom();
         getJdbcTemplate().update(INSERT_SHOW, show.id, show.name);
+        getJdbcTemplate().update(INSERT_SHOW_STATS, show.id);
     }
 
     private static final String SHOW_EXISTS =
@@ -169,6 +175,8 @@ public void delete(ShowInterface s) {
                 s.getShowId());
         getJdbcTemplate().update("DELETE FROM show_alias WHERE pk_show=?",
                 s.getShowId());
+        getJdbcTemplate().update("DELETE FROM show_stats WHERE pk_show=?",
+                s.getShowId());
         getJdbcTemplate().update("DELETE FROM show WHERE pk_show=?",
                 s.getShowId());
     }
@@ -262,7 +270,7 @@ public void updateFrameCounters(ShowInterface s, int exitStatus) {
             col = "int_frame_fail_count = int_frame_fail_count + 1";
         }
         getJdbcTemplate().update(
-                "UPDATE show SET " + col + " WHERE pk_show=?", s.getShowId());
+                "UPDATE show_stats SET " + col + " WHERE pk_show=?", s.getShowId());
     }
 }
 

cuebot/src/main/java/com/imageworks/spcue/dao/postgres/WhiteboardDaoJdbc.java

@@ -2060,7 +2060,21 @@ public Show mapRow(ResultSet rs, int rowNum) throws SQLException {
 
     private static final String GET_SHOW =
         "SELECT " +
-            "show.*," +
+            "show.pk_show," +
+            "show.str_name," +
+            "show.b_paused," +
+            "show.int_default_min_cores," +
+            "show.int_default_max_cores," +
+            "show.int_default_min_gpus," +
+            "show.int_default_max_gpus," +
+            "show.b_booking_enabled," +
+            "show.b_dispatch_enabled," +
+            "show.b_active," +
+            "show.str_comment_email," +
+            "show_stats.int_frame_insert_count," +
+            "show_stats.int_job_insert_count," +
+            "show_stats.int_frame_success_count," +
+            "show_stats.int_frame_fail_count," +
             "COALESCE(vs_show_stat.int_pending_count,0) AS int_pending_count," +
             "COALESCE(vs_show_stat.int_running_count,0) AS int_running_count," +
             "COALESCE(vs_show_stat.int_dead_count,0) AS int_dead_count," +
@@ -2069,6 +2083,7 @@ public Show mapRow(ResultSet rs, int rowNum) throws SQLException {
             "COALESCE(vs_show_stat.int_job_count,0) AS int_job_count " +
         "FROM " +
             "show " +
+        "JOIN show_stats ON (show.pk_show = show_stats.pk_show) " +
         "LEFT JOIN vs_show_stat ON (vs_show_stat.pk_show = show.pk_show) " +
         "LEFT JOIN vs_show_resource ON (vs_show_resource.pk_show=show.pk_show) " +
         "WHERE " +

cuebot/src/main/resources/conf/ddl/postgres/migrations/V16__AddShowStats.sql

@@ -0,0 +1,33 @@
+CREATE TABLE show_stats (
+    pk_show VARCHAR(36) NOT NULL,
+    int_frame_insert_count BIGINT DEFAULT 0 NOT NULL,
+    int_job_insert_count BIGINT DEFAULT 0 NOT NULL,
+    int_frame_success_count BIGINT DEFAULT 0 NOT NULL,
+    int_frame_fail_count BIGINT DEFAULT 0 NOT NULL
+);
+
+INSERT INTO show_stats (
+    pk_show,
+    int_frame_insert_count,
+    int_job_insert_count,
+    int_frame_success_count,
+    int_frame_fail_count
+) SELECT
+    pk_show,
+    int_frame_insert_count,
+    int_job_insert_count,
+    int_frame_success_count,
+    int_frame_fail_count
+  FROM show;
+
+CREATE UNIQUE INDEX c_show_stats_pk ON show_stats (pk_show);
+ALTER TABLE show_stats ADD CONSTRAINT c_show_stats_pk PRIMARY KEY
+  USING INDEX c_show_stats_pk;
+
+
+-- Destructive changes. Please test changes above prior to executing this.
+ALTER TABLE show
+    DROP COLUMN int_frame_insert_count,
+    DROP COLUMN int_job_insert_count,
+    DROP COLUMN int_frame_success_count,
+    DROP COLUMN int_frame_fail_count;

cuebot/src/main/resources/conf/ddl/postgres/seed_data.sql

@@ -1,4 +1,6 @@
-Insert into SHOW (PK_SHOW,STR_NAME,INT_DEFAULT_MAX_CORES,INT_DEFAULT_MIN_CORES,INT_FRAME_INSERT_COUNT,INT_JOB_INSERT_COUNT,INT_FRAME_SUCCESS_COUNT,INT_FRAME_FAIL_COUNT,B_BOOKING_ENABLED,B_DISPATCH_ENABLED,B_ACTIVE) values ('00000000-0000-0000-0000-000000000000', 'testing', 200000, 100, 0, 0, 0, 0, true, true, true);
+Insert into SHOW (PK_SHOW,STR_NAME,INT_DEFAULT_MAX_CORES,INT_DEFAULT_MIN_CORES,B_BOOKING_ENABLED,B_DISPATCH_ENABLED,B_ACTIVE) values ('00000000-0000-0000-0000-000000000000', 'testing', 200000, 100, true, true, true);
+
+Insert into SHOW_STATS (PK_SHOW,INT_FRAME_INSERT_COUNT,INT_JOB_INSERT_COUNT,INT_FRAME_SUCCESS_COUNT,INT_FRAME_FAIL_COUNT) values ('00000000-0000-0000-0000-000000000000',0,0,0,0);
 
 Insert into SHOW_ALIAS (PK_SHOW_ALIAS,PK_SHOW,STR_NAME) values ('00000000-0000-0000-0000-000000000001', '00000000-0000-0000-0000-000000000000', 'test');
 

cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/ShowDaoTests.java

@@ -219,20 +219,20 @@ public void testUpdateActive() {
     public void testUpdateFrameCounters() {
         ShowEntity show = showDao.findShowDetail(SHOW_NAME);
         int frameSuccess =  jdbcTemplate.queryForObject(
-                "SELECT int_frame_success_count FROM show WHERE pk_show=?",
+                "SELECT int_frame_success_count FROM show_stats WHERE pk_show=?",
                 Integer.class, show.id);
         showDao.updateFrameCounters(show, 0);
         int frameSucces2 =  jdbcTemplate.queryForObject(
-                "SELECT int_frame_success_count FROM show WHERE pk_show=?",
+                "SELECT int_frame_success_count FROM show_stats WHERE pk_show=?",
                 Integer.class, show.id);
         assertEquals(frameSuccess + 1,frameSucces2);
 
         int frameFail=  jdbcTemplate.queryForObject(
-                "SELECT int_frame_fail_count FROM show WHERE pk_show=?",
+                "SELECT int_frame_fail_count FROM show_stats WHERE pk_show=?",
                 Integer.class, show.id);
         showDao.updateFrameCounters(show, 1);
         int frameFail2 =  jdbcTemplate.queryForObject(
-                "SELECT int_frame_fail_count FROM show WHERE pk_show=?",
+                "SELECT int_frame_fail_count FROM show_stats WHERE pk_show=?",
                 Integer.class, show.id);
         assertEquals(frameFail+ 1,frameFail2);
     }

cuebot/src/test/resources/conf/ddl/postgres/test_data.sql

@@ -1,6 +1,10 @@
-Insert into SHOW (PK_SHOW,STR_NAME,INT_DEFAULT_MAX_CORES,INT_DEFAULT_MIN_CORES,INT_FRAME_INSERT_COUNT,INT_JOB_INSERT_COUNT,INT_FRAME_SUCCESS_COUNT,INT_FRAME_FAIL_COUNT,B_BOOKING_ENABLED,B_DISPATCH_ENABLED,B_ACTIVE) values ('00000000-0000-0000-0000-000000000000','pipe',20000,100,0,0,0,0,true,true,true)
+Insert into SHOW (PK_SHOW,STR_NAME,INT_DEFAULT_MAX_CORES,INT_DEFAULT_MIN_CORES,B_BOOKING_ENABLED,B_DISPATCH_ENABLED,B_ACTIVE) values ('00000000-0000-0000-0000-000000000000','pipe',20000,100,true,true,true)
 
-Insert into SHOW (PK_SHOW,STR_NAME,INT_DEFAULT_MAX_CORES,INT_DEFAULT_MIN_CORES,INT_FRAME_INSERT_COUNT,INT_JOB_INSERT_COUNT,INT_FRAME_SUCCESS_COUNT,INT_FRAME_FAIL_COUNT,B_BOOKING_ENABLED,B_DISPATCH_ENABLED,B_ACTIVE) values ('00000000-0000-0000-0000-000000000001','edu',20000,100,0,0,0,0,true,true,true)
+Insert into SHOW (PK_SHOW,STR_NAME,INT_DEFAULT_MAX_CORES,INT_DEFAULT_MIN_CORES,B_BOOKING_ENABLED,B_DISPATCH_ENABLED,B_ACTIVE) values ('00000000-0000-0000-0000-000000000001','edu',20000,100,true,true,true)
+
+Insert into SHOW_STATS (PK_SHOW,INT_FRAME_INSERT_COUNT,INT_JOB_INSERT_COUNT,INT_FRAME_SUCCESS_COUNT,INT_FRAME_FAIL_COUNT) values ('00000000-0000-0000-0000-000000000000',0,0,0,0)
+
+Insert into SHOW_STATS (PK_SHOW,INT_FRAME_INSERT_COUNT,INT_JOB_INSERT_COUNT,INT_FRAME_SUCCESS_COUNT,INT_FRAME_FAIL_COUNT) values ('00000000-0000-0000-0000-000000000001',0,0,0,0)
 
 
 Insert into SHOW_ALIAS (PK_SHOW_ALIAS,PK_SHOW,STR_NAME) values ('00000000-0000-0000-0000-000000000001','00000000-0000-0000-0000-000000000000','fx')

cuegui/tests/Constants_tests.py

@@ -25,7 +25,7 @@
 
 import mock
 import pyfakefs.fake_filesystem_unittest
-from PySide2 import QtGui
+from qtpy import QtGui
 
 import opencue
 import cuegui.Constants
@@ -40,6 +40,7 @@
 '''
 
 
+# pylint: disable=import-outside-toplevel,redefined-outer-name,reimported
 class ConstantsTests(pyfakefs.fake_filesystem_unittest.TestCase):
     def setUp(self):
         self.setUpPyfakefs()
@@ -53,6 +54,7 @@ def test__should_load_user_config_from_env_var(self):
         self.fs.create_file(config_file_path, contents=CONFIG_YAML)
         os.environ['CUEGUI_CONFIG_FILE'] = config_file_path
 
+        import cuegui.Constants
         result = importlib.reload(cuegui.Constants)
 
         self.assertEqual('98.707.68', result.VERSION)
@@ -65,6 +67,7 @@ def test__should_load_user_config_from_user_profile(self):
         config_file_path = '/home/username/.config/opencue/cuegui.yaml'
         self.fs.create_file(config_file_path, contents=CONFIG_YAML)
 
+        import cuegui.Constants
         result = importlib.reload(cuegui.Constants)
 
         self.assertEqual('98.707.68', result.VERSION)
@@ -73,6 +76,7 @@ def test__should_load_user_config_from_user_profile(self):
 
     @mock.patch('platform.system', new=mock.Mock(return_value='Linux'))
     def test__should_use_default_values(self):
+        import cuegui.Constants
         result = importlib.reload(cuegui.Constants)
 
         self.assertNotEqual('98.707.68', result.VERSION)
@@ -161,6 +165,7 @@ def test__should_use_default_values(self):
 
     @mock.patch('platform.system', new=mock.Mock(return_value='Darwin'))
     def test__should_use_mac_editor(self):
+        import cuegui.Constants
         result = importlib.reload(cuegui.Constants)
 
         self.assertEqual('open -t', result.DEFAULT_EDITOR)

requirements.txt

@@ -12,7 +12,8 @@ packaging==20.9
 pathlib==1.0.1;python_version<"3.4"
 protobuf==3.17.3;python_version<"3.0"
 psutil==5.6.7
-pyfakefs==3.6
+pyfakefs==3.6;python_version<"3.7"
+pyfakefs==5.2.3;python_version>="3.7"
 pylint==2.6.0;python_version>="3.7"
 pynput==1.7.6
 PyYAML==5.1