Optimise memory use in long array processing

local/o365/classes/feature/cohortsync/main.php

@@ -143,7 +143,13 @@ public function add_mapping(string $groupoid, int $cohortid): bool {
     public function get_mappings(): array {
         global $DB;
 
-        $mappings = $DB->get_records('local_o365_objects', ['type' => 'group', 'subtype' => 'cohort']);
+        // Use recordset instead of get_records to reduce memory usage.
+        $mappingrecordset = $DB->get_recordset('local_o365_objects', ['type' => 'group', 'subtype' => 'cohort']);
+        $mappings = [];
+        foreach ($mappingrecordset as $mapping) {
+            $mappings[$mapping->id] = $mapping;
+        }
+        $mappingrecordset->close();
 
         return $mappings;
     }
@@ -297,7 +303,13 @@ private function sync_cohort_members_by_cohort_id_and_microsoft_user_objects(int
         global $DB;
 
         $microsoftuseroids = array_column($microsoftuserobjects, 'id');
-        $currentmembers = $DB->get_records('cohort_members', ['cohortid' => $cohortid], '', 'userid');
+        // Use recordset instead of get_records to reduce memory usage.
+        $currentmembersrecordset = $DB->get_recordset('cohort_members', ['cohortid' => $cohortid], '', 'userid');
+        $currentmembers = [];
+        foreach ($currentmembersrecordset as $member) {
+            $currentmembers[$member->userid] = $member;
+        }
+        $currentmembersrecordset->close();
         $connectedusers = $this->get_all_potential_user_details($microsoftuseroids, array_keys($currentmembers));
 
         $microsoftuseroidsflipped = array_flip($microsoftuseroids);

local/o365/classes/feature/coursesync/main.php

@@ -1072,11 +1072,13 @@ public function update_teams_cache(): bool {
 
         // Build existing teams records cache.
         $this->mtrace('Building existing teams cache records', 1);
-        $existingcacherecords = $DB->get_records('local_o365_teams_cache');
+        // Use recordset instead of get_records to reduce memory usage.
+        $existingcacherecordset = $DB->get_recordset('local_o365_teams_cache');
         $existingcachebyoid = [];
-        foreach ($existingcacherecords as $existingcacherecord) {
+        foreach ($existingcacherecordset as $existingcacherecord) {
             $existingcachebyoid[$existingcacherecord->objectid] = $existingcacherecord;
         }
+        $existingcacherecordset->close();
 
         // Compare, then create, update, or delete cache.
         $this->mtrace('Updating teams cache records', 1);

local/o365/classes/feature/sds/task/sync.php

@@ -86,8 +86,9 @@ public static function runsync(unified $apiclient): bool {
                     static::mtrace('Only basic SDS profile data required', 2);
                 }
 
-                $oidcusers = $DB->get_records('user', ['auth' => 'oidc', 'deleted' => 0]);
-                foreach ($oidcusers as $userid => $oidcuser) {
+                // Use recordset instead of get_records to reduce memory usage.
+                $oidcusersrecordset = $DB->get_recordset('user', ['auth' => 'oidc', 'deleted' => 0]);
+                foreach ($oidcusersrecordset as $userid => $oidcuser) {
                     $completeuser = get_complete_user_data('id', $userid);
                     if ($completeuser) {
                         static::mtrace('Processing user ' . $oidcuser->username, 3);
@@ -178,6 +179,7 @@ public static function runsync(unified $apiclient): bool {
                         }
                     }
                 }
+                $oidcusersrecordset->close();
             } else {
                 static::mtrace('SDS field mapping disabled', 2);
             }
@@ -540,8 +542,20 @@ public static function clean_up_sds_sync_records() {
         static::mtrace('Running course sync cleanup', 1);
 
         // Get existing synced records.
-        $syncedsdsschools = $DB->get_records('local_o365_objects', ['type' => 'sdsschool']);
-        $syncedsdssections = $DB->get_records('local_o365_objects', ['type' => 'sdssection']);
+        // Use recordset instead of get_records to reduce memory usage.
+        $syncedsdsschoolsrecordset = $DB->get_recordset('local_o365_objects', ['type' => 'sdsschool']);
+        $syncedsdsschools = [];
+        foreach ($syncedsdsschoolsrecordset as $school) {
+            $syncedsdsschools[$school->id] = $school;
+        }
+        $syncedsdsschoolsrecordset->close();
+
+        $syncedsdssectionsrecordset = $DB->get_recordset('local_o365_objects', ['type' => 'sdssection']);
+        $syncedsdssections = [];
+        foreach ($syncedsdssectionsrecordset as $section) {
+            $syncedsdssections[$section->id] = $section;
+        }
+        $syncedsdssectionsrecordset->close();
 
         // Clean up schools.
         $enabledschools = get_config('local_o365', 'sdsschools');

local/o365/classes/observers.php

@@ -477,12 +477,14 @@ public static function handle_enrol_instance_updated(enrol_instance_updated $eve
             }
         }
 
-        $userenrolments = $DB->get_records('user_enrolments', ['enrolid' => $event->objectid]);
+        // Use recordset instead of get_records to reduce memory usage.
+        $userenrolmentsrecordset = $DB->get_recordset('user_enrolments', ['enrolid' => $event->objectid]);
 
-        foreach ($userenrolments as $userenrolment) {
+        foreach ($userenrolmentsrecordset as $userenrolment) {
             \local_o365\feature\coursesync\utils::sync_user_role_in_course_group($userenrolment->userid, $courseid, 0,
                 $coursegroupobjectrecordid, true);
         }
+        $userenrolmentsrecordset->close();
 
         return true;
     }

local/o365/classes/utils.php

@@ -562,10 +562,12 @@ public static function get_connected_users(): array {
 
         $connectedusers = [];
 
-        $userobjectrecords = $DB->get_records('local_o365_objects', ['type' => 'user']);
-        foreach ($userobjectrecords as $userobjectrecord) {
+        // Use recordset instead of get_records to reduce memory usage.
+        $userobjectrecordset = $DB->get_recordset('local_o365_objects', ['type' => 'user']);
+        foreach ($userobjectrecordset as $userobjectrecord) {
             $connectedusers[$userobjectrecord->moodleid] = $userobjectrecord->objectid;
         }
+        $userobjectrecordset->close();
 
         return $connectedusers;
     }
@@ -596,16 +598,18 @@ public static function update_groups_cache(unified $graphclient, int $baselevel
             return false;
         }
 
-        $existingcacherecords = $DB->get_records('local_o365_groups_cache');
+        // Use recordset instead of get_records to reduce memory usage.
+        $existingcacherecordset = $DB->get_recordset('local_o365_groups_cache');
         $existinggroupsbyoid = [];
         $existingnotfoundgroupsbyoid = [];
-        foreach ($existingcacherecords as $existingcacherecord) {
+        foreach ($existingcacherecordset as $existingcacherecord) {
             if ($existingcacherecord->not_found_since) {
                 $existingnotfoundgroupsbyoid[$existingcacherecord->objectid] = $existingcacherecord;
             } else {
                 $existinggroupsbyoid[$existingcacherecord->objectid] = $existingcacherecord;
             }
         }
+        $existingcacherecordset->close();
 
         foreach ($grouplist as $group) {
             if (array_key_exists($group['id'], $existingnotfoundgroupsbyoid)) {
@@ -623,9 +627,8 @@ public static function update_groups_cache(unified $graphclient, int $baselevel
                     $cacherecord->description = $group['description'];
                     $DB->update_record('local_o365_groups_cache', $cacherecord);
                     static::mtrace("Updated group ID {$group['id']} in cache.", $baselevel + 1);
-                } else {
-                    static::mtrace("Group ID {$group['id']} in cache is up to date.", $baselevel + 1);
                 }
+                // Removed the "up to date" message to reduce unnecessary output.
                 unset($existinggroupsbyoid[$group['id']]);
             } else {
                 $cacherecord = new stdClass();
@@ -661,17 +664,23 @@ public static function clean_up_not_found_groups(int $baselevel = 1): void {
         static::mtrace('Clean up non-existing groups from database', $baselevel);
 
         $cutofftime = strtotime('-5 minutes');
-        $sql = "SELECT *
+        $sql = "SELECT objectid
                   FROM {local_o365_groups_cache}
                  WHERE not_found_since != 0
                    AND not_found_since < :cutofftime";
         $records = $DB->get_records_sql($sql, ['cutofftime' => $cutofftime]);
 
-        foreach ($records as $record) {
-            $DB->delete_records('local_o365_groups_cache', ['objectid' => $record->objectid]);
-            $DB->delete_records('local_o365_objects', ['objectid' => $record->objectid]);
-            $DB->delete_records('local_o365_teams_cache', ['objectid' => $record->objectid]);
-            static::mtrace('Deleted non-existing group ' . $record->objectid . ' from groups cache.', $baselevel + 1);
+        if (!empty($records)) {
+            $objectids = array_keys($records);
+
+            // Use bulk delete with IN clause for better performance.
+            [$insql, $inparams] = $DB->get_in_or_equal($objectids, SQL_PARAMS_NAMED);
+            $DB->delete_records_select('local_o365_groups_cache', "objectid $insql", $inparams);
+            $DB->delete_records_select('local_o365_objects', "objectid $insql", $inparams);
+            $DB->delete_records_select('local_o365_teams_cache', "objectid $insql", $inparams);
+
+            $count = count($objectids);
+            static::mtrace("Deleted $count non-existing groups from database.", $baselevel + 1);
         }
 
         static::mtrace('Finished cleaning up non-existing groups from database.', $baselevel);

local/o365/db/install.xml

@@ -164,6 +164,9 @@
       <KEYS>
         <KEY NAME="primary" TYPE="primary" FIELDS="id"/>
       </KEYS>
+      <INDEXES>
+        <INDEX NAME="objectid" UNIQUE="false" FIELDS="objectid"/>
+      </INDEXES>
     </TABLE>
     <TABLE NAME="local_o365_groups_cache" COMMENT="Stores Groups cache. This is primarily used for cohort sync, but is used to clean up groups in local_o365_objects and teams caches too">
       <FIELDS>
@@ -176,6 +179,10 @@
       <KEYS>
         <KEY NAME="primary" TYPE="primary" FIELDS="id"/>
       </KEYS>
+      <INDEXES>
+        <INDEX NAME="objectid" UNIQUE="false" FIELDS="objectid"/>
+        <INDEX NAME="not_found_since" UNIQUE="false" FIELDS="not_found_since"/>
+      </INDEXES>
     </TABLE>
     <TABLE NAME="local_o365_course_request" COMMENT="Table to store course requests for o365">
       <FIELDS>
@@ -190,6 +197,9 @@
       <KEYS>
         <KEY NAME="primary" TYPE="primary" FIELDS="id"/>
       </KEYS>
+      <INDEXES>
+        <INDEX NAME="requeststatus" UNIQUE="false" FIELDS="requeststatus"/>
+      </INDEXES>
     </TABLE>
   </TABLES>
 </XMLDB>

local/o365/db/upgrade.php

@@ -1323,5 +1323,44 @@ function xmldb_local_o365_upgrade($oldversion) {
         upgrade_plugin_savepoint(true, 2024100711, 'local', 'o365');
     }
 
+    if ($oldversion < 2025040807) {
+        // Add index to local_o365_teams_cache.objectid for better query performance.
+        $table = new xmldb_table('local_o365_teams_cache');
+        $index = new xmldb_index('objectid', XMLDB_INDEX_NOTUNIQUE, ['objectid']);
+
+        // Conditionally launch add index objectid.
+        if (!$dbman->index_exists($table, $index)) {
+            $dbman->add_index($table, $index);
+        }
+
+        // Add indexes to local_o365_groups_cache for better query performance.
+        $table = new xmldb_table('local_o365_groups_cache');
+        $index = new xmldb_index('objectid', XMLDB_INDEX_NOTUNIQUE, ['objectid']);
+
+        // Conditionally launch add index objectid.
+        if (!$dbman->index_exists($table, $index)) {
+            $dbman->add_index($table, $index);
+        }
+
+        $index = new xmldb_index('not_found_since', XMLDB_INDEX_NOTUNIQUE, ['not_found_since']);
+
+        // Conditionally launch add index not_found_since.
+        if (!$dbman->index_exists($table, $index)) {
+            $dbman->add_index($table, $index);
+        }
+
+        // Add index to local_o365_course_request.requeststatus for better query performance.
+        $table = new xmldb_table('local_o365_course_request');
+        $index = new xmldb_index('requeststatus', XMLDB_INDEX_NOTUNIQUE, ['requeststatus']);
+
+        // Conditionally launch add index requeststatus.
+        if (!$dbman->index_exists($table, $index)) {
+            $dbman->add_index($table, $index);
+        }
+
+        // O365 savepoint reached.
+        upgrade_plugin_savepoint(true, 2025040807, 'local', 'o365');
+    }
+
     return true;
 }

local/o365/version.php

@@ -26,7 +26,7 @@
 
 defined('MOODLE_INTERNAL') || die();
 
-$plugin->version = 2025040805;
+$plugin->version = 2025040807;
 $plugin->requires = 2025040800;
 $plugin->release = '5.0.1';
 $plugin->component = 'local_o365';