diff --git a/src/agent_observer.py b/src/agent_observer.py
index 20bd1a1..27a34e5 100644
--- a/src/agent_observer.py
+++ b/src/agent_observer.py
@@ -89,7 +89,7 @@ def _on_agent_relation_changed(self, _: ops.RelationChangedEvent) -> None:
     def _on_agent_relation_departed(self, _: ops.RelationDepartedEvent) -> None:
         """Handle agent relation departed event."""
         try:
-            self.jenkins_agent_service.stop()
+            self.jenkins_agent_service.reset()
         except service.ServiceStopError:
             self.charm.unit.status = ops.BlockedStatus("Error stopping the agent service")
             return
diff --git a/src/service.py b/src/service.py
index 2432be8..e6abf94 100644
--- a/src/service.py
+++ b/src/service.py
@@ -164,8 +164,8 @@ def restart(self) -> None:
         if not self._startup_check():
             raise ServiceRestartError("Error waiting for the agent service to start")
 
-    def stop(self) -> None:
-        """Stop the agent service.
+    def reset(self) -> None:
+        """Stop the agent service and clear its configuration file.
 
         Raises:
             ServiceStopError: if systemctl stop returns a non-zero exit code.
@@ -175,6 +175,8 @@ def stop(self) -> None:
         except systemd.SystemdError as exc:
             logger.error("service %s failed to stop", AGENT_SERVICE_NAME)
             raise ServiceStopError(f"service {AGENT_SERVICE_NAME} failed to stop") from exc
+        config_file = Path(f"{SYSTEMD_SERVICE_CONF_DIR}/override.conf")
+        config_file.unlink(missing_ok=True)
 
     def _startup_check(self) -> bool:
         """Check whether the service was correctly started.
diff --git a/tests/unit/test_agent.py b/tests/unit/test_agent.py
index bc4362a..b39466b 100644
--- a/tests/unit/test_agent.py
+++ b/tests/unit/test_agent.py
@@ -5,6 +5,7 @@
 
 """Test for agent relations."""
 
+import pathlib
 from unittest.mock import MagicMock, PropertyMock
 
 import ops.testing
@@ -141,6 +142,8 @@ def test_agent_relation_departed(
     assert: The charm falls into BlockedStatus with the correct message.
     """
     monkeypatch.setattr(systemd, "service_stop", MagicMock())
+    path_unlink_mock = MagicMock()
+    monkeypatch.setattr(pathlib.Path, "unlink", path_unlink_mock)
 
     harness = harness_with_agent_relation
     harness.begin()
@@ -152,3 +155,4 @@ def test_agent_relation_departed(
     charm: JenkinsAgentCharm = harness.charm
     assert charm.unit.status.name == ops.BlockedStatus.name
     assert charm.unit.status.message == "Waiting for config/relation."
+    path_unlink_mock.assert_called_once()