Merge branch 'rest_gateway' into rest_gateway

Signed-off-by: Diego Tavares <[email protected]>

.github/workflows/sonar-cloud-pipeline.yml

@@ -10,6 +10,8 @@ jobs:
     runs-on: ubuntu-latest
     container: aswf/ci-opencue:2020
     name: Analyze Python Components
+    env:
+      ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true
     steps:
       - name: Checkout
         uses: actions/checkout@v3
@@ -31,6 +33,8 @@ jobs:
     runs-on: ubuntu-latest
     container: aswf/ci-opencue:2020
     name: Analyze Cuebot
+    env:
+      ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true
     steps:
       - name: Checkout
         uses: actions/checkout@v3

.github/workflows/testing-pipeline.yml

@@ -11,6 +11,8 @@ jobs:
     name: Run Python Unit Tests (CY2022)
     runs-on: ubuntu-latest
     container: aswf/ci-opencue:2022
+    env:
+      ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true
     steps:
       - uses: actions/checkout@v3
       - name: Run Python Tests
@@ -21,6 +23,8 @@ jobs:
     runs-on: ubuntu-latest
     container:
       image: aswf/ci-opencue:2022
+    env:
+      ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true
     steps:
       - uses: actions/checkout@v3
       - name: Build with Gradle
@@ -49,28 +53,42 @@ jobs:
           chown -R aswfuser:aswfgroup .
           su -c "cd cuebot && ./gradlew build --stacktrace --info" aswfuser
 
-  test_python2:
-    name: Run Python Unit Tests using Python2
+  test_python_2024:
+    name: Run Python Unit Tests (CY2024)
     runs-on: ubuntu-latest
-    container: aswf/ci-opencue:2019
+    container: aswf/ci-opencue:2024
     steps:
       - uses: actions/checkout@v3
       - name: Run Python Tests
         run: ci/run_python_tests.sh
 
+  test_cuebot_2024:
+    name: Build Cuebot and Run Unit Tests (CY2024)
+    runs-on: ubuntu-latest
+    container:
+      image: aswf/ci-opencue:2024
+    steps:
+      - uses: actions/checkout@v3
+      - name: Build with Gradle
+        run: |
+          chown -R aswfuser:aswfgroup .
+          su -c "cd cuebot && ./gradlew build --stacktrace --info" aswfuser
+
   test_pyside6:
     name: Run CueGUI Tests using PySide6
     runs-on: ubuntu-latest
     container: almalinux:9
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
       - name: Run CueGUI Tests
         run: ci/test_pyside6.sh
 
   lint_python:
     name: Lint Python Code
     runs-on: ubuntu-latest
     container: aswf/ci-opencue:2022
+    env:
+      ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true
     steps:
       - uses: actions/checkout@v3
       - name: Lint Python Code
@@ -93,7 +111,7 @@ jobs:
       - uses: actions/checkout@v3
       - name: Get Changed Files
         id: get_changed_files
-        uses: tj-actions/changed-files@v35
+        uses: tj-actions/changed-files@v41
       - name: Check for Version Change
         run: ci/check_changed_files.py ${{ steps.get_changed_files.outputs.modified_files }} ${{ steps.get_changed_files.outputs.deleted_files }}
 
@@ -112,6 +130,6 @@ jobs:
       - uses: actions/checkout@v3
       - name: Get Changed Files
         id: get_changed_files
-        uses: tj-actions/changed-files@v35
+        uses: tj-actions/changed-files@v41
       - name: Check for Version Change
         run: ci/check_version_bump.py ${{ steps.get_changed_files.outputs.all_changed_and_modified_files }}

.gitignore

@@ -9,28 +9,9 @@ venv*/
 .coverage
 coverage.xml
 htmlcov/
-/.eggs/
-*.iml
-.venv/
-*_pb2.py
-*_pb2_grpc.py
-svn/*
-/dist/
-cueweb/.next/*
-cueweb/node_modules/*
-.metals/*
-rqd/.eggs/*
-rqd/deploy/rpm/RPMS/x86_64/*
-pycue/.metals/metals.h2.db
-pycue/.metals/metals.lock.db
-pycue/.metals/metals.log
-pycue/.vscode/settings.json
+rest_gateway/*.tar\.gz
+/.env
+.envrc
 .vscode
-.bloop
-cuebot/gradle/wrapper/gradle-6.0.1-bin.zip
-*.class
-bin/
-logs/
-opencue.code-workspace
-stack_test.yml
-rest_gateway/*.tar\.gz
+.venv/
+.eggs/*

CODEOWNERS

@@ -1 +1 @@
-*          @bcipriano @gregdenton @jrray @smith1511 @larsbijl @DiegoTavares @IdrisMiles @splhack
+*          @bcipriano @DiegoTavares @ramonfigueiredo

README.md

@@ -7,6 +7,7 @@
 - [Introduction](#Introduction)
 - [OpenCue features](#OpenCue-features)
 - [Learn more](#Learn-more)
+- [Meeting notes](#Meeting-notes)
 - [Contact us](#Contact-us)
 
 # Introduction
@@ -39,6 +40,12 @@ OpenCue provides the following features to help manage rendering jobs at scale:
 For more information on installing, using, and administering OpenCue, visit
 [www.opencue.io](https://www.opencue.io).
 
+# Meeting notes
+
+Starting from May 2024, all Opencue meeting notes are stored on the [Opencue Confluence page](http://wiki.aswf.io/display/OPENCUE/OpenCue+Home).
+
+For meeting notes before May 2024, please refer to the Opencue repository in the [opencue/tsc/meetings](https://github.com/AcademySoftwareFoundation/OpenCue/tree/master/tsc/meetings) folder.
+
 # Contact us
 
 To join the OpenCue discussion forum for users and admins, join the

VERSION.in

@@ -1 +1 @@
-0.22
+0.34

ci/pylintrc_main

@@ -77,7 +77,8 @@ disable=c-extension-no-member,
         too-many-public-methods,
         too-many-return-statements,
         too-many-statements,
-        useless-object-inheritance
+        useless-object-inheritance,
+        consider-using-f-string
 
 # Enable the message, report, category or checker with the given id(s). You can
 # either give multiple identifier separated by comma (,) or put this option
@@ -202,7 +203,12 @@ ignored-classes=optparse.Values,thread._local,_thread._local
 # (useful for modules/projects where namespaces are manipulated during runtime
 # and thus existing member attributes cannot be deduced by static analysis). It
 # supports qualified module names, as well as Unix pattern matching.
-ignored-modules=
+ignored-modules=opencue.compiled_proto,
+            opencue.compiled_proto.filter_pb2,
+            opencue.compiled_proto.host_pb2,
+            rqd.compiled_proto.rqd_pb2,
+            rqd.compiled_proto.host_pb2,
+            rqd.compiled_proto.report_pb2
 
 # Show a hint with possible names when a member name was not found. The aspect
 # of finding the hint is based on edit distance.
@@ -523,5 +529,5 @@ min-public-methods=2
 
 # Exceptions that will emit a warning when being caught. Defaults to
 # "BaseException, Exception".
-overgeneral-exceptions=BaseException,
-                       Exception
+overgeneral-exceptions=builtins.BaseException,
+                       builtins.Exception

ci/pylintrc_test

@@ -77,7 +77,8 @@ disable=arguments-differ,
         too-many-statements,
         unused-argument,
         unused-variable,
-        useless-object-inheritance
+        useless-object-inheritance,
+        consider-using-f-string
 
 # Enable the message, report, category or checker with the given id(s). You can
 # either give multiple identifier separated by comma (,) or put this option
@@ -202,7 +203,12 @@ ignored-classes=optparse.Values,thread._local,_thread._local
 # (useful for modules/projects where namespaces are manipulated during runtime
 # and thus existing member attributes cannot be deduced by static analysis). It
 # supports qualified module names, as well as Unix pattern matching.
-ignored-modules=
+ignored-modules=opencue.compiled_proto,
+                opencue.compiled_proto.filter_pb2,
+                opencue.compiled_proto.host_pb2,
+                rqd.compiled_proto.rqd_pb2,
+                rqd.compiled_proto.host_pb2,
+                rqd.compiled_proto.report_pb2
 
 # Show a hint with possible names when a member name was not found. The aspect
 # of finding the hint is based on edit distance.
@@ -523,5 +529,5 @@ min-public-methods=2
 
 # Exceptions that will emit a warning when being caught. Defaults to
 # "BaseException, Exception".
-overgeneral-exceptions=BaseException,
-                       Exception
+overgeneral-exceptions=builtins.BaseException,
+                       builtins.Exception

ci/run_integration_test.sh

@@ -86,7 +86,7 @@ wait_for_service_state() {
             log ERROR "Timed out waiting for Docker compose to come up"
             exit 1
         fi
-        container=$(docker compose ps --all --format json | jq ".[] | select(.Service==\"$1\")")
+        container=$(docker compose ps --all --format json | jq -s ".[] | select(.Service==\"$1\")")
         if [[ ${container} = "" ]]; then
             log INFO "Service \"$1\": no container yet"
         else
@@ -102,7 +102,7 @@ wait_for_service_state() {
 }
 
 verify_flyway_success() {
-    container=$(docker compose ps --all --format json | jq '.[] | select(.Service=="flyway")')
+    container=$(docker compose ps --all --format json | jq -s '.[] | select(.Service=="flyway")')
     container_name=$(echo "$container" | jq -r '.Name')
     exit_code=$(echo "$container" | jq -r '.ExitCode')
     if [[ ${exit_code} = 0 ]]; then
@@ -143,8 +143,7 @@ test_pycue() {
         exit 1
     fi
 
-    rqd_name=$(docker compose ps --format json | jq -r '.[] | select(.Service=="rqd") | .Name')
-    rqd_ip=$(docker inspect -f '{{range.NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "${rqd_name}")
+    rqd_ip=$(docker inspect -f '{{range.NetworkSettings.Networks}}{{.IPAddress}}{{end}}' opencue-rqd-1)
     want_hosts="['${rqd_ip}']"
     got_hosts=$(python -c 'import opencue; print([host.name() for host in opencue.api.getHosts()])')
     if [[ "${got_hosts}" = "${want_hosts}" ]]; then
@@ -169,8 +168,7 @@ test_cueadmin() {
         exit 1
     fi
 
-    rqd_name=$(docker compose ps --format json | jq -r '.[] | select(.Service=="rqd") | .Name')
-    want_host=$(docker inspect -f '{{range.NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "${rqd_name}")
+    want_host=$(docker inspect -f '{{range.NetworkSettings.Networks}}{{.IPAddress}}{{end}}' opencue-rqd-1)
     lh_response=$(cueadmin -lh)
     got_host=$(echo "${lh_response}" | tail -n 1 | cut -d ' ' -f 1)
     if [[ "${got_host}" = "${want_host}" ]]; then

cueadmin/cueadmin/common.py

@@ -289,6 +289,7 @@ def _convert(val):
         else:
             result = opencue.api.criterion_pb2.GreaterThanFloatSearchCriterion(
                 value=_convert(mixed))
+    # pylint: disable=use-a-generator
     elif any([isinstance(mixed.__class__, crit_cls) for crit_cls in criterions]):
         result = mixed
     elif not mixed:
@@ -336,6 +337,7 @@ def _convert(val):
         else:
             result = opencue.api.criterion_pb2.GreaterThanIntegerSearchCriterion(
                 value=_convert(mixed))
+    # pylint: disable=use-a-generator
     elif any([isinstance(mixed.__class__, crit_cls) for crit_cls in criterions]):
         result = mixed
     elif not mixed:
@@ -562,7 +564,7 @@ def handleArgs(args):
 
     if args.facility:
         logger.debug("setting facility to %s", args.facility)
-        opencue.Cuebot.setFacility(args.facility)
+        opencue.Cuebot.setHostWithFacility(args.facility)
 
     #
     # Query

cueadmin/tests/common_tests.py

@@ -72,7 +72,7 @@ def testSetServer(self, setHostsMock):
 
         setHostsMock.assert_called_with([serverName])
 
-    @mock.patch('opencue.Cuebot.setFacility')
+    @mock.patch('opencue.Cuebot.setHostWithFacility')
     def testSetFacility(self, setFacilityMock):
         args = self.parser.parse_args(['-facility', TEST_FACILITY])
 

cuebot/build.gradle

@@ -61,6 +61,9 @@ dependencies {
     compile group: 'org.apache.logging.log4j', name: 'log4j-api', version: '2.16.0'
     compile group: 'org.apache.logging.log4j', name: 'log4j-core', version: '2.16.0'
     compile group: 'org.apache.logging.log4j', name: 'log4j-slf4j-impl', version: '2.16.0'
+    compile group: 'io.sentry', name: 'sentry-log4j2', version: '7.11.0'
+    compile group: 'io.prometheus', name: 'simpleclient', version: '0.16.0'
+    compile group: 'io.prometheus', name: 'simpleclient_servlet', version: '0.16.0'
 
     protobuf fileTree("../proto/")
 

cuebot/src/main/java/com/imageworks/spcue/ActionEntity.java

@@ -85,6 +85,10 @@ public String getActionId() {
     }
 
     public String getFilterId() {
+        if (filterId == null){
+            throw new SpcueRuntimeException(
+                    "Trying to get a filterId from a ActityEntity created without a filter");
+        }
         return filterId;
     }
 

cuebot/src/main/java/com/imageworks/spcue/DispatchFrame.java

@@ -47,6 +47,7 @@ public class DispatchFrame extends FrameEntity implements FrameInterface {
     public int maxGpus;
     public long minGpuMemory;
 
+    // A comma separated list of services
     public String services;
 }
 

cuebot/src/main/java/com/imageworks/spcue/LayerDetail.java

@@ -46,6 +46,7 @@ public class LayerDetail extends LayerEntity implements LayerInterface {
     public Set<String> tags = new LinkedHashSet<String>();
     public Set<String> services = new LinkedHashSet<String>();
     public Set<String> limits = new LinkedHashSet<String>();
+    public Set<String> outputs = new LinkedHashSet<String>();
 
     /*
      *