From 0f850e2d63549b19b69008a50b2ffbb3c7a778d7 Mon Sep 17 00:00:00 2001
From: DocGarbanzo <dirk.prange@web.de>
Date: Sat, 28 Sep 2024 16:01:55 +0100
Subject: [PATCH] Restrict logging messages to ERROR level for tf and
 matplotlib. Switch kivy logging to python base logging.

---
 donkeycar/management/makemovie.py       |  7 ++++---
 donkeycar/management/ui/car_screen.py   | 19 +++++++++++--------
 donkeycar/management/ui/common.py       | 12 ++++++++----
 donkeycar/management/ui/pilot_screen.py | 17 ++++++++++-------
 donkeycar/management/ui/train_screen.py | 17 +++++++++++------
 donkeycar/management/ui/tub_screen.py   | 14 ++++++++------
 donkeycar/management/ui/ui.py           | 10 ++++++----
 donkeycar/parts/interpreter.py          |  3 ++-
 donkeycar/parts/keras.py                |  5 +++--
 donkeycar/parts/keras_2.py              |  1 +
 donkeycar/pipeline/training.py          |  1 +
 donkeycar/utils.py                      |  1 +
 12 files changed, 66 insertions(+), 41 deletions(-)

diff --git a/donkeycar/management/makemovie.py b/donkeycar/management/makemovie.py
index dbb3e0f27..dcce11537 100755
--- a/donkeycar/management/makemovie.py
+++ b/donkeycar/management/makemovie.py
@@ -1,12 +1,13 @@
 import tempfile
+import logging
 
+logging.getLogger('tensorflow').setLevel(logging.WARNING)
+import tensorflow as tf
 from tensorflow.python.keras import activations
 from tensorflow.python.keras import backend as K
 from tensorflow.python.keras.models import load_model
-import tensorflow as tf
-import cv2
 from matplotlib import cm
-
+import cv2
 
 import donkeycar as dk
 from donkeycar.parts.tub_v2 import Tub
diff --git a/donkeycar/management/ui/car_screen.py b/donkeycar/management/ui/car_screen.py
index 4676f6a56..ae1ac8183 100644
--- a/donkeycar/management/ui/car_screen.py
+++ b/donkeycar/management/ui/car_screen.py
@@ -2,8 +2,8 @@
 import re
 from functools import partial
 from subprocess import Popen, PIPE, STDOUT
+import logging
 
-from kivy import Logger
 from kivy.clock import Clock
 from kivy.properties import NumericProperty, ObjectProperty, StringProperty, \
     ListProperty, BooleanProperty
@@ -12,6 +12,9 @@
 from donkeycar.management.ui.rc_file_handler import rc_handler
 
 
+logger = logging.getLogger(__name__)
+
+
 class CarScreen(AppScreen):
     """ Screen for interacting with the car. """
     config = ObjectProperty(force_dispatch=True, allownone=True)
@@ -55,7 +58,7 @@ def pull(self, tub_dir):
         if not self.ids.create_dir.active:
             target += '/'
         cmd = ['rsync', '-rv', '--progress', '--partial', target, dest]
-        Logger.info('car pull: ' + str(cmd))
+        logger.info('car pull: ' + str(cmd))
         proc = Popen(cmd, shell=False, stdout=PIPE, text=True,
                      encoding='utf-8', universal_newlines=True)
         repeats = 100
@@ -84,7 +87,7 @@ def send_pilot(self):
         dest = f'{self.config.PI_USERNAME}@{self.config.PI_HOSTNAME}:' + \
                f'{os.path.join(self.car_dir, "models")}'
         cmd = ['rsync', '-rv', '--progress', '--partial', *filter, src, dest]
-        Logger.info('car push: ' + ' '.join(cmd))
+        logger.info('car push: ' + ' '.join(cmd))
         proc = Popen(cmd, shell=False, stdout=PIPE,
                      encoding='utf-8', universal_newlines=True)
         repeats = 0
@@ -189,7 +192,7 @@ def drive(self):
                f'{self.config.PI_USERNAME}@{self.config.PI_HOSTNAME}',
                f'source env/bin/activate; cd {self.car_dir}; ./manage.py '
                f'drive {model_args} 2>&1']
-        Logger.info(f'car connect: {cmd}')
+        logger.info(f'car connect: {cmd}')
         proc = Popen(cmd, shell=False, stdout=PIPE, text=True,
                      encoding='utf-8', universal_newlines=True)
         while True:
@@ -201,12 +204,12 @@ def drive(self):
                 if res:
                     try:
                         self.pid = int(res.group(0).split('PID: ')[1])
-                        Logger.info(f'car connect: manage.py drive PID: '
+                        logger.info(f'car connect: manage.py drive PID: '
                                     f'{self.pid}')
                     except Exception as e:
-                        Logger.error(f'car connect: {e}')
+                        logger.error(f'car connect: {e}')
                     return
-                Logger.info(f'car connect: {stdout_data}')
+                logger.info(f'car connect: {stdout_data}')
             else:
                 return
 
@@ -215,5 +218,5 @@ def stop(self):
             cmd = f'ssh {self.config.PI_USERNAME}@{self.config.PI_HOSTNAME} '\
                   + f'kill -SIGINT {self.pid}'
             out = os.popen(cmd).read()
-            Logger.info(f"car connect: Kill PID {self.pid} + {out}")
+            logger.info(f"car connect: Kill PID {self.pid} + {out}")
             self.pid = None
diff --git a/donkeycar/management/ui/common.py b/donkeycar/management/ui/common.py
index 3cd7c38e3..f41d99341 100644
--- a/donkeycar/management/ui/common.py
+++ b/donkeycar/management/ui/common.py
@@ -2,12 +2,13 @@
 import os
 import io
 import time
+import logging
 
 import numpy as np
 from functools import partial
 from PIL import Image as PilImage
 
-from kivy import Logger
+#from kivy import logger
 from kivy.app import App
 from kivy.clock import Clock
 from kivy.core.window import Window
@@ -26,6 +27,9 @@
 
 from donkeycar.management.ui.rc_file_handler import rc_handler
 
+logger = logging.getLogger(__name__)
+
+
 LABEL_SPINNER_TEXT = 'Add/remove'
 
 
@@ -139,7 +143,7 @@ def update(self, record):
             return
         field, index = decompose(self.field)
         if not field in record.underlying:
-            Logger.error(f'Record: Bad record {record.underlying["_index"]} - '
+            logger.error(f'Record: Bad record {record.underlying["_index"]} - '
                          f'missing field {self.field}')
             return
         val = record.underlying[field]
@@ -236,10 +240,10 @@ def update(self, record):
             self.core_image = CoreImage(bytes_io, ext='png')
             self.texture = self.core_image.texture
         except KeyError as e:
-            Logger.error(f'Record {record.underlying["_index"]}: '
+            logger.error(f'Record {record.underlying["_index"]}: '
                          f'Missing key: {e}')
         except Exception as e:
-            Logger.error(f'Record : {record.underlying["_index"]}'
+            logger.error(f'Record : {record.underlying["_index"]}'
                          f'Bad record: {e}')
 
     def get_image(self, record):
diff --git a/donkeycar/management/ui/pilot_screen.py b/donkeycar/management/ui/pilot_screen.py
index 05cff213f..ca072ccd7 100644
--- a/donkeycar/management/ui/pilot_screen.py
+++ b/donkeycar/management/ui/pilot_screen.py
@@ -1,7 +1,8 @@
 from copy import copy #, deepcopy
 import os
+import logging
 
-from kivy import Logger
+#from kivy import logger
 from kivy.properties import StringProperty, ObjectProperty, ListProperty, \
     NumericProperty, BooleanProperty
 from kivy.uix.boxlayout import BoxLayout
@@ -18,6 +19,8 @@
 from donkeycar.utils import get_model_by_type
 from donkeycar.parts.keras_2 import KerasSquarePlusImu, KerasSquarePlusMemoryLap
 
+logger = logging.getLogger(__name__)
+
 
 ALL_FILTERS = ['*.h5', '*.tflite', '*.savedmodel', '*.trt']
 
@@ -43,12 +46,12 @@ def remove_pilot_from_db(entry):
                 # if successfully loaded, add to rc file
                 if entry not in rc_handler.data['pilots']:
                     rc_handler.data['pilots'].append(entry)
-                Logger.info(f'Pilot: Successfully loaded {self.file_path}')
+                logger.info(f'Pilot: Successfully loaded {self.file_path}')
             except FileNotFoundError:
-                Logger.error(f'Pilot: Model {self.file_path} not found')
+                logger.error(f'Pilot: Model {self.file_path} not found')
                 remove_pilot_from_db(entry)
             except Exception as e:
-                Logger.error(f'Failed loading {self.file_path}: {e}')
+                logger.error(f'Failed loading {self.file_path}: {e}')
                 remove_pilot_from_db(entry)
 
     def on_model_type(self, obj, model_type):
@@ -131,7 +134,7 @@ def get_image(self, record):
             # Not each model is supported in each interpreter
             output = self.pilot_loader.pilot.run(*args)
         except Exception as e:
-            Logger.error(e)
+            logger.error(e)
 
         rgb = (0, 0, 255)
         MakeMovie.draw_line_into_image(output[0], output[1], True, img_arr, rgb)
@@ -196,7 +199,7 @@ def open_popup(self):
         popup.open()
 
     def on_selected(self, obj, select):
-        Logger.info(f"Selected {select}")
+        logger.info(f"Selected {select}")
         if self.is_post:
             self.pilot_screen.post_trans_list = self.selected
         else:
@@ -292,7 +295,7 @@ def on_config(self, obj, cfg):
             for c in self.ids.pilot_board.children:
                 c.ids.pilot_loader.root_path = self.config.MODELS_PATH
         except Exception as e:
-            Logger.error(f'Error at config update in train screen: {e}')
+            logger.error(f'Error at config update in train screen: {e}')
 
     def initialise(self, e):
         # self.ids.pilot_board.add_viewer()
diff --git a/donkeycar/management/ui/train_screen.py b/donkeycar/management/ui/train_screen.py
index 5bdce73e4..7ef7c40d0 100644
--- a/donkeycar/management/ui/train_screen.py
+++ b/donkeycar/management/ui/train_screen.py
@@ -4,7 +4,8 @@
 import json
 
 import pandas as pd
-from kivy import Logger
+import logging
+
 from kivy.clock import Clock
 from kivy.properties import ObjectProperty, NumericProperty, ListProperty, \
     StringProperty
@@ -13,9 +14,11 @@
 from kivy.uix.label import Label
 from kivy.uix.popup import Popup
 from kivy_garden.matplotlib import FigureCanvasKivyAgg
+logging.getLogger('matplotlib').setLevel(logging.ERROR)
 import matplotlib as mpl
 import matplotlib.pyplot as plt
 
+
 from donkeycar.config import Config
 from donkeycar.management.ui.common import FileChooserBase, get_app_screen, \
     AppScreen, status
@@ -24,6 +27,8 @@
 from donkeycar.pipeline.training import train
 
 
+logger = logging.getLogger(__name__)
+
 mpl.rcParams.update({'font.size': 8})
 plt.style.use('dark_background')
 fig1, ax1 = plt.subplots()
@@ -144,7 +149,7 @@ def _config_to_dict(self):
             s = s.replace("None", "null")
             cfg_list = json.loads(s)
         except Exception as e:
-            Logger.error(f'Failed json read of config: {e}')
+            logger.error(f'Failed json read of config: {e}')
         assert isinstance(cfg_list, list), "De-jsonised config should be list"
         return dict(cfg_list)
 
@@ -177,7 +182,7 @@ def on_df(self, e=None, z=None):
         # arrange subplots
         non_val_cols = [c for c in loss_df.columns if c[:4] != 'val_']
         if len(non_val_cols) != n / 2:
-            Logger.Error(f"Issue with history data, validation data history "
+            logger.Error(f"Issue with history data, validation data history "
                          f"is not half of the loss data")
             return
         subplots = [(nv, f'val_{nv}') for nv in non_val_cols]
@@ -217,7 +222,7 @@ def train_call(self, *args):
                             transfer=transfer_model,
                             comment=self.ids.comment.text)
         except Exception as e:
-            Logger.error(e)
+            logger.error(e)
             status(f'Training failed see console')
 
     def train(self):
@@ -298,7 +303,7 @@ def show_config(self, obj=None):
         pilot = self.ids.select_spinner.text
         cfg = self.database.get_entry(pilot).get('Config')
         if not cfg:
-            Logger.Error(f'Config for pilot {pilot} not found in database')
+            logger.Error(f'Config for pilot {pilot} not found in database')
             return
         popup = ConfigViewerPopup(config=cfg, title=f'Config for {pilot}')
         popup.fill_grid()
@@ -308,7 +313,7 @@ def show_history(self, obj=None):
         pilot = self.ids.select_spinner.text
         history = self.database.get_entry(pilot).get('History')
         if not history:
-            Logger.Error(f'History for pilot {pilot} not found in database')
+            logger.Error(f'History for pilot {pilot} not found in database')
             return
         df = pd.DataFrame(history)
         popup = HistoryViewerPopup(df=df, title=f'Training history for {pilot}')
diff --git a/donkeycar/management/ui/tub_screen.py b/donkeycar/management/ui/tub_screen.py
index 7cf27879a..1bc46fc31 100644
--- a/donkeycar/management/ui/tub_screen.py
+++ b/donkeycar/management/ui/tub_screen.py
@@ -4,16 +4,16 @@
 import plotly.express as px
 import pandas as pd
 
-
-from kivy import Logger
 from kivy.uix.boxlayout import BoxLayout
 from kivy.properties import NumericProperty, ObjectProperty, StringProperty, \
     ListProperty, BooleanProperty
 from kivy_garden.matplotlib.backend_kivyagg import FigureCanvasKivyAgg
+
+import logging
+logging.getLogger('matplotlib').setLevel(logging.ERROR)
 import matplotlib as mpl
 import matplotlib.pyplot as plt
 
-
 from donkeycar import load_config
 from donkeycar.management.ui.common import FileChooserBase, \
     PaddedBoxLayout, decompose, get_app_screen, BackgroundBoxLayout, AppScreen, \
@@ -23,6 +23,8 @@
 from donkeycar.pipeline.types import TubRecord
 
 
+logger = logging.getLogger(__name__)
+
 mpl.rcParams.update({'font.size': 8})
 plt.style.use('dark_background')
 fig, ax = plt.subplots()
@@ -47,11 +49,11 @@ def load_action(self):
             # If load successful, store into app config
             rc_handler.data['car_dir'] = self.file_path
         except FileNotFoundError:
-            Logger.error(f'Config: Directory {self.file_path} has no '
+            logger.error(f'Config: Directory {self.file_path} has no '
                          f'config.py')
             self.config = None
         except Exception as e:
-            Logger.error(f'Config: {e}')
+            logger.error(f'Config: {e}')
             self.config = None
 
     def on_config(self, obj, cfg):
@@ -118,7 +120,7 @@ def select(underlying):
                     res = train_filter(record)
                     return res
                 except KeyError as err:
-                    Logger.error(f'Filter: {err}')
+                    logger.error(f'Filter: {err}')
                     return True
 
         self.records = [TubRecord(cfg, self.tub.base_path, record)
diff --git a/donkeycar/management/ui/ui.py b/donkeycar/management/ui/ui.py
index ee6db4586..46b77c051 100644
--- a/donkeycar/management/ui/ui.py
+++ b/donkeycar/management/ui/ui.py
@@ -1,5 +1,7 @@
 import os
-from kivy.logger import Logger, LOG_LEVELS
+import logging
+
+#from kivy.logger import Logger, LOG_LEVELS
 from kivy.clock import Clock
 from kivy.app import App
 from kivy.properties import StringProperty
@@ -14,8 +16,8 @@
 from donkeycar.management.ui.tub_screen import TubScreen
 from donkeycar.management.ui.common import AppScreen
 
-Logger.setLevel(LOG_LEVELS["info"])
-
+# Logger.setLevel(LOG_LEVELS["info"])
+logger = logging.getLogger(__name__)
 Window.size = (800, 800)
 
 
@@ -55,7 +57,7 @@ def on_stop(self, *args):
         tub = self.root.ids.tub_screen.ids.tub_loader.tub
         if tub:
             tub.close()
-        Logger.info("App: Good bye Donkey")
+        logger.info("App: Good bye Donkey")
 
 
 def main():
diff --git a/donkeycar/parts/interpreter.py b/donkeycar/parts/interpreter.py
index 3d9ed0f27..f37fb4b5b 100755
--- a/donkeycar/parts/interpreter.py
+++ b/donkeycar/parts/interpreter.py
@@ -2,8 +2,9 @@
 from abc import ABC, abstractmethod
 import logging
 import numpy as np
-from typing import Union, Sequence, List
+from typing import Union, Sequence
 
+logging.getLogger('tensorflow').setLevel(logging.WARNING)
 import tensorflow as tf
 from tensorflow import keras
 from tensorflow.python.saved_model import tag_constants, signature_constants
diff --git a/donkeycar/parts/keras.py b/donkeycar/parts/keras.py
index e14ff6c33..6f5f4b2fd 100644
--- a/donkeycar/parts/keras.py
+++ b/donkeycar/parts/keras.py
@@ -15,7 +15,7 @@
 
 import numpy as np
 from typing import Dict, Tuple, Optional, Union, List, Sequence, Callable, Any
-from logging import getLogger
+import logging
 
 from tensorflow.python.data.ops.dataset_ops import DatasetV1, DatasetV2
 
@@ -24,6 +24,7 @@
 from donkeycar.pipeline.types import TubRecord
 from donkeycar.parts.interpreter import Interpreter, KerasInterpreter
 
+logging.getLogger('tensorflow').setLevel(logging.WARNING)
 import tensorflow as tf
 from tensorflow import keras
 from tensorflow.keras.layers import (Dense, Input,Convolution2D,
@@ -42,7 +43,7 @@
 XY = Union[float, np.ndarray, Tuple[Union[float, np.ndarray], ...]]
 
 
-logger = getLogger(__name__)
+logger = logging.getLogger(__name__)
 
 
 class KerasPilot(ABC):
diff --git a/donkeycar/parts/keras_2.py b/donkeycar/parts/keras_2.py
index 74386b221..09f629d99 100644
--- a/donkeycar/parts/keras_2.py
+++ b/donkeycar/parts/keras_2.py
@@ -5,6 +5,7 @@
 
 import numpy as np
 from typing import Dict, Tuple, Union, List, Callable
+logging.getLogger('tensorflow').setLevel(logging.WARNING)
 import tensorflow as tf
 
 from tensorflow import keras
diff --git a/donkeycar/pipeline/training.py b/donkeycar/pipeline/training.py
index 6b49b2e46..15785a077 100644
--- a/donkeycar/pipeline/training.py
+++ b/donkeycar/pipeline/training.py
@@ -16,6 +16,7 @@
 from donkeycar.pipeline.augmentations import ImageAugmentation
 from donkeycar.parts.image_transformations import ImageTransformations
 from donkeycar.utils import get_model_by_type, normalize_image, train_test_split
+logging.getLogger('tensorflow').setLevel(logging.WARNING)
 import tensorflow as tf
 import numpy as np
 
diff --git a/donkeycar/utils.py b/donkeycar/utils.py
index ccc46f28d..7ecbac2db 100644
--- a/donkeycar/utils.py
+++ b/donkeycar/utils.py
@@ -20,6 +20,7 @@
 from typing import List, Any, Tuple, Union
 from collections import UserString
 
+logging.getLogger('PIL').setLevel(logging.ERROR)
 from PIL import Image
 import numpy as np