Minor updates to answer relevance

Signed-off-by: Fred Reiss <[email protected]>

Author: frreiss

mellea/backends/huggingface.py

@@ -311,6 +311,17 @@ async def _generate_from_intrinsic(
             "messages": conversation,
             "extra_body": {"documents": docs},
         }
+
+        # Convert other parameters from Mellea proprietary format to standard format.
+        for model_option in model_options:
+            if model_option == ModelOption.TEMPERATURE:
+                request_json["temperature"] = model_options[model_option]
+            else:
+                raise ValueError(
+                    f"Model option '{model_option}' not implemented "
+                    f"on this generation code path."
+                )
+
         rewritten = rewriter.transform(request_json, **action.intrinsic_kwargs)
 
         # TODO: Handle caching here. granite_common doesn't tell us what changed,

mellea/backends/openai.py

@@ -435,6 +435,17 @@ async def _generate_from_intrinsic(
             "extra_body": {"documents": docs},
         }
 
+        # Convert other parameters from Mellea proprietary format to standard format.
+        if model_options is not None:
+            for model_option in model_options:
+                if model_option == ModelOption.TEMPERATURE:
+                    request_json["temperature"] = model_options[model_option]
+                else:
+                    raise ValueError(
+                        f"Model option '{model_option}' not implemented "
+                        f"on this generation code path."
+                    )
+
         rewritten = rewriter.transform(request_json, **action.intrinsic_kwargs)
 
         self.load_adapter(adapter.qualified_name)

mellea/stdlib/intrinsics/rag.py

@@ -9,6 +9,7 @@
     AdapterType,
     GraniteCommonAdapter,
 )
+from mellea.backends.types import ModelOption
 from mellea.stdlib.base import ChatContext, Document
 from mellea.stdlib.chat import Message
 from mellea.stdlib.intrinsics.intrinsic import Intrinsic
@@ -64,6 +65,7 @@ def _call_intrinsic(
         context,
         backend,
         # No rejection sampling, please
+        model_options={ModelOption.TEMPERATURE: 0.0},
         strategy=None,
     )
 
@@ -277,7 +279,7 @@ def rewrite_answer_for_relevance(
         backend,
         kwargs={
             "answer_relevance_category": result_json["answer_relevance_category"],
-            "answer_relevance_analysis": result_json["answer_relevance_category"],
+            "answer_relevance_analysis": result_json["answer_relevance_analysis"],
             "correction_method": correction_method,
         },
     )

test/stdlib_intrinsics/test_rag/test_rag.py

@@ -161,7 +161,13 @@ def test_hallucination_detection(backend):
 def test_answer_relevance(backend):
     """Verify that the answer relevance composite intrinsic functions properly."""
     context, answer, docs = _read_input_json("answer_relevance.json")
-    expected_rewrite = "Alice, Bob, and Carol attended the meeting."
+
+    # Note that this is not the optimal answer. This test is currently using an
+    # outdated LoRA adapter. Releases of new adapters will come after the Mellea
+    # integration has stabilized.
+    expected_rewrite = (
+        "The documents do not provide information about the attendees of the meeting."
+    )
 
     # First call triggers adapter loading
     result = rag.rewrite_answer_for_relevance(answer, docs, context, backend)
@@ -178,5 +184,17 @@ def test_answer_relevance(backend):
     assert result == answer
 
 
+def test_answer_relevance_classifier(backend):
+    """Verify that the first phase of the answer relevance flow behaves as expectee."""
+    context, answer, docs = _read_input_json("answer_relevance.json")
+
+    result_json = rag._call_intrinsic(
+        "answer_relevance_classifier",
+        context.add(Message("assistant", answer, documents=list(docs))),
+        backend,
+    )
+    assert result_json["answer_relevance_likelihood"] == 0.0
+
+
 if __name__ == "__main__":
     pytest.main([__file__])