Merge branch 'develop' into chore/development/update-core-files-to-new-angular-api

Author: PaRangger

.github/PULL_REQUEST_TEMPLATE.md

@@ -111,10 +111,10 @@ Prerequisites:
 - [ ] Test 2
 
 ### Test Coverage
-<!-- Please add the test coverages for all changed files here. You can see this when executing the tests locally (see build.gradle and package.json) or when looking into the corresponding Bamboo build plan. -->
-<!-- The line coverage must be above 90% for changes files and you must use extensive and useful assertions for server tests and expect statements for client tests. -->
-<!-- Note: Use the table below and confirm in the last column that you have implemented extensive assertions for server tests and expect statements for client tests. -->
-<!--       You can use `supporting_script/generate_code_cov_table/generate_code_cov_table.py` to automatically generate one from the corresponding Bamboo build plan artefacts. -->
+<!-- Please add the test coverages for all changed files modified in this PR here. You can use `supporting_script/generate_code_cov_table/generate_code_cov_table.py` to automatically generate the coverage table from the corresponding artefacts of your branch (follow the ReadMe for setup details). -->
+<!-- Alternatively you can execute the tests locally (see build.gradle and package.json) or look into the corresponding artefacts. -->
+<!-- The line coverage must be above 90% for changes files, and you must use extensive and useful assertions for server tests and expect statements for client tests. -->
+<!-- Note: Confirm in the last column that you have implemented extensive assertions for server tests and expect statements for client tests. -->
 <!--       Remove rows with only trivial changes from the table. -->
 <!--
 | Class/File | Line Coverage | Confirmation (assert/expect) |

src/main/java/de/tum/cit/aet/artemis/buildagent/dto/BuildJobQueueItem.java

@@ -48,4 +48,9 @@ public BuildJobQueueItem(BuildJobQueueItem queueItem, ResultDTO submissionResult
         this(queueItem.id(), queueItem.name(), queueItem.buildAgent(), queueItem.participationId(), queueItem.courseId(), queueItem.exerciseId(), queueItem.retryCount(),
                 queueItem.priority(), queueItem.status(), queueItem.repositoryInfo(), queueItem.jobTimingInfo(), queueItem.buildConfig(), submissionResult);
     }
+
+    public BuildJobQueueItem(BuildJobQueueItem queueItem, BuildAgentDTO buildAgent, int newRetryCount) {
+        this(queueItem.id(), queueItem.name(), buildAgent, queueItem.participationId(), queueItem.courseId(), queueItem.exerciseId(), newRetryCount, queueItem.priority(), null,
+                queueItem.repositoryInfo(), new JobTimingInfo(queueItem.jobTimingInfo.submissionDate(), ZonedDateTime.now(), null), queueItem.buildConfig(), null);
+    }
 }

src/main/java/de/tum/cit/aet/artemis/buildagent/service/BuildJobExecutionService.java

@@ -378,18 +378,18 @@ private BuildResult parseTestResults(TarArchiveInputStream testResultsTarInputSt
             }
 
             // Read the contents of the tar entry as a string.
-            String xmlString = readTarEntryContent(testResultsTarInputStream);
+            String fileString = readTarEntryContent(testResultsTarInputStream);
             // Get the file name of the tar entry.
             String fileName = getFileName(tarEntry);
 
             try {
                 // Check if the file is a static code analysis report file
                 if (StaticCodeAnalysisTool.getToolByFilePattern(fileName).isPresent()) {
-                    processStaticCodeAnalysisReportFile(fileName, xmlString, staticCodeAnalysisReports, buildJobId);
+                    processStaticCodeAnalysisReportFile(fileName, fileString, staticCodeAnalysisReports, buildJobId);
                 }
                 else {
                     // ugly workaround because in swift result files \n\t breaks the parsing
-                    var testResultFileString = xmlString.replace("\n\t", "");
+                    var testResultFileString = fileString.replace("\n\t", "");
                     if (!testResultFileString.isBlank()) {
                         processTestResultFile(testResultFileString, failedTests, successfulTests);
                     }
@@ -418,7 +418,7 @@ private boolean isValidTestResultFile(TarArchiveEntry tarArchiveEntry) {
         String result = (lastIndexOfSlash != -1 && lastIndexOfSlash + 1 < name.length()) ? name.substring(lastIndexOfSlash + 1) : name;
 
         // Java test result files are named "TEST-*.xml", Python test result files are named "*results.xml".
-        return !tarArchiveEntry.isDirectory() && result.endsWith(".xml") && !result.equals("pom.xml");
+        return !tarArchiveEntry.isDirectory() && (result.endsWith(".xml") && !result.equals("pom.xml") || result.endsWith(".sarif"));
     }
 
     /**
@@ -444,12 +444,12 @@ private String getFileName(TarArchiveEntry tarEntry) {
      * Processes a static code analysis report file and adds the report to the corresponding list.
      *
      * @param fileName                  the file name of the static code analysis report file
-     * @param xmlString                 the content of the static code analysis report file
+     * @param reportContent             the content of the static code analysis report file
      * @param staticCodeAnalysisReports the list of static code analysis reports
      */
-    private void processStaticCodeAnalysisReportFile(String fileName, String xmlString, List<StaticCodeAnalysisReportDTO> staticCodeAnalysisReports, String buildJobId) {
+    private void processStaticCodeAnalysisReportFile(String fileName, String reportContent, List<StaticCodeAnalysisReportDTO> staticCodeAnalysisReports, String buildJobId) {
         try {
-            staticCodeAnalysisReports.add(ReportParser.getReport(xmlString, fileName));
+            staticCodeAnalysisReports.add(ReportParser.getReport(reportContent, fileName));
         }
         catch (UnsupportedToolException e) {
             String msg = "Failed to parse static code analysis report for " + fileName;

src/main/java/de/tum/cit/aet/artemis/buildagent/service/SharedQueueProcessingService.java

@@ -266,16 +266,26 @@ private void checkAvailabilityAndProcessNextBuild() {
             processBuild(buildJob);
         }
         catch (RejectedExecutionException e) {
-            log.error("Couldn't add build job to threadpool: {}\n Concurrent Build Jobs Count: {} Active tasks in pool: {}, Concurrent Build Jobs Size: {}", buildJob,
+            // TODO: we should log this centrally and not on the local node
+            log.error("Couldn't add build job to thread pool: {}\n Concurrent Build Jobs Count: {} Active tasks in pool: {}, Concurrent Build Jobs Size: {}", buildJob,
                     localProcessingJobs.get(), localCIBuildExecutorService.getActiveCount(), localCIBuildExecutorService.getMaximumPoolSize(), e);
 
             // Add the build job back to the queue
             if (buildJob != null) {
                 processingJobs.remove(buildJob.id());
 
-                buildJob = new BuildJobQueueItem(buildJob, new BuildAgentDTO("", "", ""));
-                log.info("Adding build job back to the queue: {}", buildJob);
-                queue.add(buildJob);
+                // At most try out the build job 5 times when they get rejected
+                if (buildJob.retryCount() >= 5) {
+                    // TODO: we should log this centrally and not on the local node
+                    log.error("Build job was rejected 5 times. Not adding build job back to the queue: {}", buildJob);
+                }
+                else {
+                    // NOTE: we increase the retry count here, because the build job was not processed successfully
+                    // TODO: we should try to run this job on a different build agent to avoid getting the same error again
+                    buildJob = new BuildJobQueueItem(buildJob, new BuildAgentDTO("", "", ""), buildJob.retryCount() + 1);
+                    log.info("Adding build job {} back to the queue with retry count {}", buildJob, buildJob.retryCount());
+                    queue.add(buildJob);
+                }
                 localProcessingJobs.decrementAndGet();
             }
 
@@ -360,7 +370,9 @@ private BuildAgentInformation getUpdatedLocalBuildAgentInformation(BuildJobQueue
     }
 
     private List<BuildJobQueueItem> getProcessingJobsOfNode(String memberAddress) {
-        return processingJobs.values().stream().filter(job -> Objects.equals(job.buildAgent().memberAddress(), memberAddress)).toList();
+        // NOTE: we should not use streams with IMap, because it can be unstable, when many items are added at the same time and there is a slow network condition
+        List<BuildJobQueueItem> processingJobsList = new ArrayList<>(processingJobs.values());
+        return processingJobsList.stream().filter(job -> Objects.equals(job.buildAgent().memberAddress(), memberAddress)).toList();
     }
 
     private void removeOfflineNodes() {
@@ -549,7 +561,8 @@ private void resumeBuildAgent() {
     private boolean nodeIsAvailable() {
         log.debug("Currently processing jobs on this node: {}, active threads in Pool: {}, maximum pool size of thread executor : {}", localProcessingJobs.get(),
                 localCIBuildExecutorService.getActiveCount(), localCIBuildExecutorService.getMaximumPoolSize());
-        return localProcessingJobs.get() < localCIBuildExecutorService.getMaximumPoolSize();
+        return localProcessingJobs.get() < localCIBuildExecutorService.getMaximumPoolSize()
+                && localCIBuildExecutorService.getActiveCount() < localCIBuildExecutorService.getMaximumPoolSize() && localCIBuildExecutorService.getQueue().isEmpty();
     }
 
     public class QueuedBuildJobItemListener implements ItemListener<BuildJobQueueItem> {

src/main/java/de/tum/cit/aet/artemis/core/config/MetricsBean.java

@@ -290,21 +290,21 @@ private void registerLocalCIMetrics() {
     }
 
     private static int extractRunningBuilds(Optional<SharedQueueManagementService> sharedQueueManagementService) {
-        return sharedQueueManagementService.map(queueManagementService -> queueManagementService.getBuildAgentInformation().stream()
-                .map(buildAgentInformation -> buildAgentInformation.runningBuildJobs().size()).reduce(0, Integer::sum)).orElse(0);
+        return sharedQueueManagementService.map(SharedQueueManagementService::getProcessingJobsSize).orElse(0);
     }
 
     private static int extractQueuedBuilds(Optional<SharedQueueManagementService> sharedQueueManagementService) {
-        return sharedQueueManagementService.map(queueManagementService -> queueManagementService.getQueuedJobs().size()).orElse(0);
+        return sharedQueueManagementService.map(SharedQueueManagementService::getQueuedJobsSize).orElse(0);
     }
 
     private static int extractBuildAgents(Optional<SharedQueueManagementService> sharedQueueManagementService) {
-        return sharedQueueManagementService.map(queueManagementService -> queueManagementService.getBuildAgentInformation().size()).orElse(0);
+        return sharedQueueManagementService.map(SharedQueueManagementService::getBuildAgentInformationSize).orElse(0);
     }
 
     private static int extractMaxConcurrentBuilds(Optional<SharedQueueManagementService> sharedQueueManagementService) {
         return sharedQueueManagementService.map(queueManagementService -> queueManagementService.getBuildAgentInformation().stream()
-                .map(BuildAgentInformation::maxNumberOfConcurrentBuildJobs).reduce(0, Integer::sum)).orElse(0);
+                .filter(agent -> agent.status() != BuildAgentInformation.BuildAgentStatus.PAUSED).map(BuildAgentInformation::maxNumberOfConcurrentBuildJobs)
+                .reduce(0, Integer::sum)).orElse(0);
     }
 
     private void registerWebsocketMetrics() {

src/main/java/de/tum/cit/aet/artemis/core/web/admin/AdminBuildJobQueueResource.java

@@ -195,7 +195,7 @@ public ResponseEntity<BuildJobsStatisticsDTO> getBuildJobStatistics(@RequestPara
     }
 
     /**
-     * {@code PUT /api/admin/agent/{agentName}/pause} : Pause the specified build agent.
+     * {@code PUT /api/admin/agents/{agentName}/pause} : Pause the specified build agent.
      * This endpoint allows administrators to pause a specific build agent by its name.
      * Pausing a build agent will prevent it from picking up any new build jobs until it is resumed.
      *
@@ -207,15 +207,34 @@ public ResponseEntity<BuildJobsStatisticsDTO> getBuildJobStatistics(@RequestPara
      * @return {@link ResponseEntity} with status code 204 (No Content) if the agent was successfully paused
      *         or an appropriate error response if something went wrong
      */
-    @PutMapping("agent/{agentName}/pause")
+    @PutMapping("agents/{agentName}/pause")
     public ResponseEntity<Void> pauseBuildAgent(@PathVariable String agentName) {
         log.debug("REST request to pause agent {}", agentName);
         localCIBuildJobQueueService.pauseBuildAgent(agentName);
         return ResponseEntity.noContent().build();
     }
 
     /**
-     * {@code PUT /api/admin/agent/{agentName}/resume} : Resume the specified build agent.
+     * {@code PUT /api/admin/agents/pause-all} : Pause all build agents.
+     * This endpoint allows administrators to pause all build agents.
+     * Pausing all build agents will prevent them from picking up any new build jobs until they are resumed.
+     *
+     * <p>
+     * <strong>Authorization:</strong> This operation requires admin privileges, enforced by {@code @EnforceAdmin}.
+     * </p>
+     *
+     * @return {@link ResponseEntity} with status code 204 (No Content) if all agents were successfully paused
+     *         or an appropriate error response if something went wrong
+     */
+    @PutMapping("agents/pause-all")
+    public ResponseEntity<Void> pauseAllBuildAgents() {
+        log.debug("REST request to pause all agents");
+        localCIBuildJobQueueService.pauseAllBuildAgents();
+        return ResponseEntity.noContent().build();
+    }
+
+    /**
+     * {@code PUT /api/admin/agents/{agentName}/resume} : Resume the specified build agent.
      * This endpoint allows administrators to resume a specific build agent by its name.
      * Resuming a build agent will allow it to pick up new build jobs again.
      *
@@ -227,10 +246,29 @@ public ResponseEntity<Void> pauseBuildAgent(@PathVariable String agentName) {
      * @return {@link ResponseEntity} with status code 204 (No Content) if the agent was successfully resumed
      *         or an appropriate error response if something went wrong
      */
-    @PutMapping("agent/{agentName}/resume")
+    @PutMapping("agents/{agentName}/resume")
     public ResponseEntity<Void> resumeBuildAgent(@PathVariable String agentName) {
         log.debug("REST request to resume agent {}", agentName);
         localCIBuildJobQueueService.resumeBuildAgent(agentName);
         return ResponseEntity.noContent().build();
     }
+
+    /**
+     * {@code PUT /api/admin/agents/resume-all} : Resume all build agents.
+     * This endpoint allows administrators to resume all build agents.
+     * Resuming all build agents will allow them to pick up new build jobs again.
+     *
+     * <p>
+     * <strong>Authorization:</strong> This operation requires admin privileges, enforced by {@code @EnforceAdmin}.
+     * </p>
+     *
+     * @return {@link ResponseEntity} with status code 204 (No Content) if all agents were successfully resumed
+     *         or an appropriate error response if something went wrong
+     */
+    @PutMapping("agents/resume-all")
+    public ResponseEntity<Void> resumeAllBuildAgents() {
+        log.debug("REST request to resume all agents");
+        localCIBuildJobQueueService.resumeAllBuildAgents();
+        return ResponseEntity.noContent().build();
+    }
 }

src/main/java/de/tum/cit/aet/artemis/programming/domain/StaticCodeAnalysisTool.java

@@ -1,7 +1,8 @@
 package de.tum.cit.aet.artemis.programming.domain;
 
-import java.util.ArrayList;
+import java.util.EnumMap;
 import java.util.List;
+import java.util.Map;
 import java.util.Objects;
 import java.util.Optional;
 
@@ -10,28 +11,29 @@
  */
 public enum StaticCodeAnalysisTool {
 
-    SPOTBUGS(ProgrammingLanguage.JAVA, "spotbugs:spotbugs", "spotbugsXml.xml"), CHECKSTYLE(ProgrammingLanguage.JAVA, "checkstyle:checkstyle", "checkstyle-result.xml"),
-    PMD(ProgrammingLanguage.JAVA, "pmd:pmd", "pmd.xml"), PMD_CPD(ProgrammingLanguage.JAVA, "pmd:cpd", "cpd.xml"), SWIFTLINT(ProgrammingLanguage.SWIFT, "", "swiftlint-result.xml"),
-    GCC(ProgrammingLanguage.C, "", "gcc.xml");
+    // @formatter:off
+    SPOTBUGS("spotbugsXml.xml"),
+    CHECKSTYLE("checkstyle-result.xml"),
+    PMD("pmd.xml"),
+    PMD_CPD("cpd.xml"),
+    SWIFTLINT("swiftlint-result.xml"),
+    GCC("gcc.xml"),
+    OTHER(null),
+    ;
+    // @formatter:on
 
-    private final ProgrammingLanguage language;
+    // @formatter:off
+    private static final Map<ProgrammingLanguage, List<StaticCodeAnalysisTool>> TOOLS_OF_PROGRAMMING_LANGUAGE = new EnumMap<>(Map.of(
+        ProgrammingLanguage.JAVA, List.of(SPOTBUGS, CHECKSTYLE, PMD, PMD_CPD),
+        ProgrammingLanguage.SWIFT, List.of(SWIFTLINT),
+        ProgrammingLanguage.C, List.of(GCC)
+    ));
+    // @formatter:on
 
-    private final String command;
+    private final String fileName;
 
-    private final String filePattern;
-
-    StaticCodeAnalysisTool(ProgrammingLanguage language, String command, String filePattern) {
-        this.language = language;
-        this.command = command;
-        this.filePattern = filePattern;
-    }
-
-    public String getTask() {
-        return this.command;
-    }
-
-    public String getFilePattern() {
-        return this.filePattern;
+    StaticCodeAnalysisTool(String fileName) {
+        this.fileName = fileName;
     }
 
     /**
@@ -41,13 +43,7 @@ public String getFilePattern() {
      * @return List of static code analysis
      */
     public static List<StaticCodeAnalysisTool> getToolsForProgrammingLanguage(ProgrammingLanguage language) {
-        List<StaticCodeAnalysisTool> tools = new ArrayList<>();
-        for (var tool : StaticCodeAnalysisTool.values()) {
-            if (tool.language == language) {
-                tools.add(tool);
-            }
-        }
-        return tools;
+        return TOOLS_OF_PROGRAMMING_LANGUAGE.getOrDefault(language, List.of());
     }
 
     /**
@@ -58,7 +54,7 @@ public static List<StaticCodeAnalysisTool> getToolsForProgrammingLanguage(Progra
      */
     public static Optional<StaticCodeAnalysisTool> getToolByFilePattern(String fileName) {
         for (StaticCodeAnalysisTool tool : StaticCodeAnalysisTool.values()) {
-            if (Objects.equals(fileName, tool.filePattern)) {
+            if (Objects.equals(fileName, tool.fileName)) {
                 return Optional.of(tool);
             }
         }