From 0c4c8bb21bf856b3c6a2711e4686eadaf0207376 Mon Sep 17 00:00:00 2001 From: Ethan Ma Date: Thu, 18 Jul 2024 09:42:25 -0700 Subject: [PATCH 1/3] Add eval after all to mosaicmllogger --- composer/loggers/mosaicml_logger.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/composer/loggers/mosaicml_logger.py b/composer/loggers/mosaicml_logger.py index d7c83b85fa..0441e76afb 100644 --- a/composer/loggers/mosaicml_logger.py +++ b/composer/loggers/mosaicml_logger.py @@ -138,6 +138,11 @@ def fit_end(self, state: State, logger: Logger) -> None: def eval_end(self, state: State, logger: Logger) -> None: self._flush_metadata(force_flush=True) + def eval_after_all(self, state: State, logger: Logger) -> None: + # Log model evaluation finished time for run events + self.log_metadata({'eval_after_all_time': time.time()}) + self._flush_metadata(force_flush=True) + def predict_end(self, state: State, logger: Logger) -> None: self._flush_metadata(force_flush=True) From 1a574fcd197c0a2a69c4c3d416b78fec9a4f2360 Mon Sep 17 00:00:00 2001 From: Ethan Ma Date: Fri, 26 Jul 2024 08:58:49 -0700 Subject: [PATCH 2/3] add test --- tests/loggers/test_mosaicml_logger.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/loggers/test_mosaicml_logger.py b/tests/loggers/test_mosaicml_logger.py index e308dab122..9fb976202f 100644 --- a/tests/loggers/test_mosaicml_logger.py +++ b/tests/loggers/test_mosaicml_logger.py @@ -322,6 +322,7 @@ def test_run_events_logged(monkeypatch): assert 'mosaicml/training_sub_progress' not in metadata assert isinstance(metadata['mosaicml/train_finished_time'], float) assert isinstance(metadata['mosaicml/train_started_time'], float) + assert isinstance(metadata['mosaicml/eval_after_all_time'], float) def test_token_training_progress_metrics(): From b091dff2595ca7c424986ba222a4c5726345e87c Mon Sep 17 00:00:00 2001 From: Ethan Ma Date: Fri, 26 Jul 2024 11:05:27 -0700 Subject: [PATCH 3/3] force flush --- composer/loggers/mosaicml_logger.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/composer/loggers/mosaicml_logger.py b/composer/loggers/mosaicml_logger.py index 8495c347b5..b47c476e48 100644 --- a/composer/loggers/mosaicml_logger.py +++ b/composer/loggers/mosaicml_logger.py @@ -145,8 +145,7 @@ def eval_end(self, state: State, logger: Logger) -> None: def eval_after_all(self, state: State, logger: Logger) -> None: # Log model evaluation finished time for run events - self.log_metadata({'eval_after_all_time': time.time()}) - self._flush_metadata(force_flush=True) + self.log_metadata({'eval_after_all_time': time.time()}, force_flush=True) def predict_end(self, state: State, logger: Logger) -> None: self._flush_metadata(force_flush=True)