Skip to content

Commit

Permalink
Fixed tests
Browse files Browse the repository at this point in the history
  • Loading branch information
@aelaguiz
aelaguiz committed Mar 18, 2024
1 parent d631942 commit 04333c9
Showing 1 changed file with 4 additions and 31 deletions.
35 changes: 4 additions & 31 deletions tests/test_manual_examples.py
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,3 @@
# tests/test_prompt_signature_examples.py
import pytest
from langdspy.field_descriptors import InputField, OutputField, HintField
from langdspy.prompt_strategies import PromptSignature, DefaultPromptStrategy
Expand All @@ -8,24 +7,20 @@ class TestPromptSignature(PromptSignature):
input = InputField(name="input", desc="Input field")
output = OutputField(name="output", desc="Output field")
hint = HintField(desc="Hint field")

__examples__ = [
({"input": "Example input 1"}, "Example output 1"),
({"input": "Example input 2"}, "Example output 2"),
]

def test_format_prompt_with_examples_openai():
prompt_runner = PromptRunner(template_class=TestPromptSignature, prompt_strategy=DefaultPromptStrategy)

formatted_prompt = prompt_runner.template._format_openai_prompt(
trained_state=None,
use_training=True,
examples=TestPromptSignature.__examples__,
input="Test input"
)

print(formatted_prompt)

assert "💡 Hint field" in formatted_prompt
assert "✅input: Input field" in formatted_prompt
assert "🔑output: Output field" in formatted_prompt
Expand All @@ -38,17 +33,15 @@ def test_format_prompt_with_examples_openai():

def test_format_prompt_with_examples_anthropic():
prompt_runner = PromptRunner(template_class=TestPromptSignature, prompt_strategy=DefaultPromptStrategy)

formatted_prompt = prompt_runner.template._format_anthropic_prompt(
trained_state=None,
use_training=True,
examples=TestPromptSignature.__examples__,
input="Test input"
)

assert "<hint>Hint field</hint>" in formatted_prompt
assert "<input>: Input field" in formatted_prompt
assert "<output>: Output field" in formatted_prompt
assert "<input>Input field</input>" in formatted_prompt
assert "<output>Output field</output>" in formatted_prompt
assert "<input>Example input 1</input>" in formatted_prompt
assert "<output>Example output 1</output>" in formatted_prompt
assert "<input>Example input 2</input>" in formatted_prompt
Expand All @@ -58,14 +51,12 @@ def test_format_prompt_with_examples_anthropic():

def test_format_prompt_without_examples_openai():
prompt_runner = PromptRunner(template_class=TestPromptSignature, prompt_strategy=DefaultPromptStrategy)

formatted_prompt = prompt_runner.template._format_openai_prompt(
trained_state=None,
use_training=False,
examples=[],
input="Test input"
)

assert "💡 Hint field" in formatted_prompt
assert "✅input: Input field" in formatted_prompt
assert "🔑output: Output field" in formatted_prompt
Expand All @@ -78,17 +69,15 @@ def test_format_prompt_without_examples_openai():

def test_format_prompt_without_examples_anthropic():
prompt_runner = PromptRunner(template_class=TestPromptSignature, prompt_strategy=DefaultPromptStrategy)

formatted_prompt = prompt_runner.template._format_anthropic_prompt(
trained_state=None,
use_training=False,
examples=[],
input="Test input"
)

assert "<hint>Hint field</hint>" in formatted_prompt
assert "<input>: Input field" in formatted_prompt
assert "<output>: Output field" in formatted_prompt
assert "<input>Input field</input>" in formatted_prompt
assert "<output>Output field</output>" in formatted_prompt
assert "<input>Test input</input>" in formatted_prompt
assert "<output></output>" in formatted_prompt
assert "Example input 1" not in formatted_prompt
Expand All @@ -102,35 +91,29 @@ class ValidPromptSignature(PromptSignature):
input2 = InputField(name="input2", desc="Input field 2")
output1 = OutputField(name="output1", desc="Output field 1")
output2 = OutputField(name="output2", desc="Output field 2")

__examples__ = [
({"input1": "Example input 1", "input2": "Example input 2"}, {"output1": "Example output 1", "output2": "Example output 2"}),
]

prompt_runner = PromptRunner(template_class=ValidPromptSignature, prompt_strategy=DefaultPromptStrategy)
prompt_runner.template.validate_examples() # Should not raise any exception

def test_validate_examples_invalid_input_field():
class InvalidInputPromptSignature(PromptSignature):
input = InputField(name="input", desc="Input field")
output = OutputField(name="output", desc="Output field")

__examples__ = [
({"invalid_input": "Example input"}, "Example output"),
]

with pytest.raises(ValueError, match="Example input field 'invalid_input' not found in input_variables"):
PromptRunner(template_class=InvalidInputPromptSignature, prompt_strategy=DefaultPromptStrategy)

def test_validate_examples_invalid_output_field():
class InvalidOutputPromptSignature(PromptSignature):
input = InputField(name="input", desc="Input field")
output = OutputField(name="output", desc="Output field")

__examples__ = [
({"input": "Example input"}, {"invalid_output": "Example output"}),
]

with pytest.raises(ValueError, match="Example output field 'invalid_output' not found in output_variables"):
PromptRunner(template_class=InvalidOutputPromptSignature, prompt_strategy=DefaultPromptStrategy)

Expand All @@ -139,11 +122,9 @@ class InvalidOutputFormatPromptSignature(PromptSignature):
input = InputField(name="input", desc="Input field")
output1 = OutputField(name="output1", desc="Output field 1")
output2 = OutputField(name="output2", desc="Output field 2")

__examples__ = [
({"input": "Example input"}, "Example output"),
]

with pytest.raises(ValueError, match="Example output must be a dictionary when there are multiple output fields"):
PromptRunner(template_class=InvalidOutputFormatPromptSignature, prompt_strategy=DefaultPromptStrategy)

Expand All @@ -152,20 +133,16 @@ class MultipleOutputPromptSignature(PromptSignature):
input = InputField(name="input", desc="Input field")
output1 = OutputField(name="output1", desc="Output field 1")
output2 = OutputField(name="output2", desc="Output field 2")

__examples__ = [
({"input": "Example input"}, {"output1": "Example output 1", "output2": "Example output 2"}),
]

prompt_runner = PromptRunner(template_class=MultipleOutputPromptSignature, prompt_strategy=DefaultPromptStrategy)

formatted_prompt = prompt_runner.template._format_openai_prompt(
trained_state=None,
use_training=True,
examples=MultipleOutputPromptSignature.__examples__,
input="Test input"
)

assert "✅input: Example input" in formatted_prompt
assert "🔑output1: Example output 1" in formatted_prompt
assert "🔑output2: Example output 2" in formatted_prompt
Expand All @@ -175,20 +152,16 @@ class MultipleOutputPromptSignature(PromptSignature):
input = InputField(name="input", desc="Input field")
output1 = OutputField(name="output1", desc="Output field 1")
output2 = OutputField(name="output2", desc="Output field 2")

__examples__ = [
({"input": "Example input"}, {"output1": "Example output 1", "output2": "Example output 2"}),
]

prompt_runner = PromptRunner(template_class=MultipleOutputPromptSignature, prompt_strategy=DefaultPromptStrategy)

formatted_prompt = prompt_runner.template._format_anthropic_prompt(
trained_state=None,
use_training=True,
examples=MultipleOutputPromptSignature.__examples__,
input="Test input"
)

assert "<input>Example input</input>" in formatted_prompt
assert "<output1>Example output 1</output1>" in formatted_prompt
assert "<output2>Example output 2</output2>" in formatted_prompt

0 comments on commit 04333c9

Please sign in to comment.