Check if CometLogger experiment is alive (#19915)

Co-authored-by: Etay Livne <[email protected]>

src/lightning/pytorch/loggers/comet.py

@@ -268,7 +268,7 @@ def experiment(self) -> Union["Experiment", "ExistingExperiment", "OfflineExperi
             self.logger.experiment.some_comet_function()
 
         """
-        if self._experiment is not None:
+        if self._experiment is not None and self._experiment.alive:
             return self._experiment
 
         if self._future_experiment_key is not None:

tests/tests_pytorch/loggers/test_comet.py

@@ -66,6 +66,20 @@ def test_comet_logger_online(comet_mock):
     api.assert_called_once_with("rest")
 
 
+@mock.patch.dict(os.environ, {})
+def test_comet_experiment_resets_if_not_alive(comet_mock):
+    """Test that the CometLogger creates a new experiment if the old one is not alive anymore."""
+    logger = CometLogger()
+    assert logger._experiment is None
+    alive_experiment = Mock(alive=True)
+    logger._experiment = alive_experiment
+    assert logger.experiment is alive_experiment
+
+    unalive_experiment = Mock(alive=False)
+    logger._experiment = unalive_experiment
+    assert logger.experiment is not unalive_experiment
+
+
 @mock.patch.dict(os.environ, {})
 def test_comet_logger_no_api_key_given(comet_mock):
     """Test that CometLogger fails to initialize if both api key and save_dir are missing."""