Trace memory usage with logging (#2708)

Trace memory usage with logging

Sadly py-spy was only a time profiler and I wasn't able to find something that does not need changes in the code.
I followed mainly this guide and this celery issue suggestions.
Deep profiling will be done just in stg, quicker resource check also in prod.

Reviewed-by: Matej Focko
Reviewed-by: Laura Barcziová
Reviewed-by: Maja Massarini

packit_service/worker/handlers/abstract.py

@@ -6,7 +6,9 @@
 """
 
 import enum
+import gc
 import logging
+import resource
 import shutil
 from collections import defaultdict
 from datetime import datetime
@@ -18,6 +20,7 @@
 from celery.canvas import Signature
 from ogr.abstract import GitProject
 from packit.config import JobConfig, JobType, PackageConfig
+from packit.config.common_package_config import Deployment
 from packit.constants import DATETIME_FORMAT
 
 from packit_service.config import ServiceConfig
@@ -237,16 +240,34 @@ def get_tag_info(self) -> dict:
             tags.update({"package_name": self.data.event_dict["package_name"]})
         return tags
 
+    def log_memory_stats(self) -> None:
+        gc.collect()
+        memory_usage = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss
+        logger.info(f"Memory usage: {memory_usage} (kb)")
+
     def run_n_clean(self) -> TaskResults:
         try:
             with push_scope_to_sentry() as scope:
                 for k, v in self.get_tag_info().items():
                     scope.set_tag(k, v)
+
+                self.log_memory_stats()
+                if self.service_config.deployment in [Deployment.stg, Deployment.dev]:
+                    from guppy import hpy
+
+                    hp = hpy()
+                    before = hp.heap()
+
                 return self.run()
         except Exception as ex:
-            logger.error(f"Failed to run the handler: {ex}")
+            logger.info(f"Failed to run the handler: {ex}")
             raise
         finally:
+            self.log_memory_stats()
+            if self.service_config.deployment in [Deployment.stg, Deployment.dev]:
+                after = hp.heap()
+                leftover = after - before
+                logger.info(f"Objects still in memory generated by task execution: {leftover}")
             self.clean()
 
     def _clean_workplace(self):

tests/integration/test_dg_pr.py

@@ -8,9 +8,7 @@
 from celery.canvas import Signature
 from flexmock import flexmock
 from packit.api import PackitAPI
-from packit.config import (
-    JobConfigTriggerType,
-)
+from packit.config import Deployment, JobConfigTriggerType
 from packit.local_project import LocalProjectBuilder
 from packit.utils import commands
 
@@ -53,6 +51,7 @@ def test_downstream_koji_scratch_build(distgit_pr_event):
             command_handler_work_dir=SANDCASTLE_WORK_DIR,
             repository_cache="/tmp/repository-cache",
             add_repositories_to_repository_cache=False,
+            deployment=Deployment.stg,
         )
         .should_receive("get_project")
         .and_return(dg_project)

tests/integration/test_listen_to_fedmsg.py

@@ -15,6 +15,7 @@
 from ogr.utils import RequestResponse
 from packit.config import (
     CommonPackageConfig,
+    Deployment,
     JobConfig,
     JobConfigTriggerType,
     JobType,
@@ -2575,6 +2576,7 @@ def test_koji_build_end_downstream(
             enabled_projects_for_fedora_ci="https://src.fedoraproject.org/rpms/packit",
             koji_logs_url="",
             koji_web_url="",
+            deployment=Deployment.stg,
         )
         .should_receive("get_project")
         .and_return(flexmock(namespace="rpms", repo="packit"))