Skip to content

Commit

Permalink
It is more correct to call it a comparison, not just an example
Browse files Browse the repository at this point in the history
  • Loading branch information
dustalov committed Jul 27, 2024
1 parent 90460a2 commit e75f5f5
Show file tree
Hide file tree
Showing 2 changed files with 79 additions and 79 deletions.
28 changes: 14 additions & 14 deletions conftest.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,8 +15,8 @@
from hypothesis.strategies import DrawFn


class Example(NamedTuple):
"""A tuple holding example data."""
class Comparison(NamedTuple):
"""A tuple holding comparison data."""

xs: list[str] | pd.Series[str]
ys: list[str] | pd.Series[str]
Expand All @@ -28,10 +28,10 @@ def enumerate_sizes(n: int) -> list[tuple[int, ...]]:


@composite
def elements(
def comparisons(
draw: DrawFn,
shape: Literal["good", "bad"] = "good",
) -> Example: # type: ignore[type-var]
) -> Comparison: # type: ignore[type-var]
length = draw(st.integers(0, 5))

if shape == "good":
Expand All @@ -47,11 +47,11 @@ def elements(
ys = st.lists(st.text(max_size=length_y), min_size=length_y, max_size=length_y)
ws = st.lists(st.sampled_from(evalica.WINNERS), min_size=length_z, max_size=length_z)

return Example(xs=draw(xs), ys=draw(ys), ws=draw(ws))
return Comparison(xs=draw(xs), ys=draw(ys), ws=draw(ws))


@pytest.fixture()
def simple() -> Example:
def simple() -> Comparison:
df_simple = pd.read_csv(Path(__file__).resolve().parent / "simple.csv", dtype=str)

xs = df_simple["left"]
Expand All @@ -62,7 +62,7 @@ def simple() -> Example:
"tie": evalica.Winner.Draw,
})

return Example(xs=xs, ys=ys, ws=ws)
return Comparison(xs=xs, ys=ys, ws=ws)


@pytest.fixture()
Expand All @@ -75,7 +75,7 @@ def simple_golden() -> pd.DataFrame:


@pytest.fixture()
def food() -> Example:
def food() -> Comparison:
df_food = pd.read_csv(Path(__file__).resolve().parent / "food.csv", dtype=str)

xs = df_food["left"]
Expand All @@ -86,7 +86,7 @@ def food() -> Example:
"tie": evalica.Winner.Draw,
})

return Example(xs=xs, ys=ys, ws=ws)
return Comparison(xs=xs, ys=ys, ws=ws)


@pytest.fixture()
Expand All @@ -99,7 +99,7 @@ def food_golden() -> pd.DataFrame:


@pytest.fixture()
def llmfao() -> Example:
def llmfao() -> Comparison:
df_llmfao = pd.read_csv("https://github.com/dustalov/llmfao/raw/master/crowd-comparisons.csv", dtype=str)

xs = df_llmfao["left"]
Expand All @@ -110,7 +110,7 @@ def llmfao() -> Example:
"tie": evalica.Winner.Draw,
})

return Example(xs=xs, ys=ys, ws=ws)
return Comparison(xs=xs, ys=ys, ws=ws)


@pytest.fixture()
Expand All @@ -126,14 +126,14 @@ def llmfao_golden() -> pd.DataFrame:


@pytest.fixture()
def example(request: TopRequest, dataset: str) -> Example:
def comparison(request: TopRequest, dataset: str) -> Comparison:
assert dataset in DATASETS, f"unknown dataset: {dataset}"

return cast(Example, request.getfixturevalue(dataset))
return cast(Comparison, request.getfixturevalue(dataset))


@pytest.fixture()
def example_golden(request: TopRequest, dataset: str, algorithm: str) -> pd.Series[str]:
def comparison_golden(request: TopRequest, dataset: str, algorithm: str) -> pd.Series[str]:
assert dataset in DATASETS, f"unknown dataset: {dataset}"

df_golden = cast(pd.DataFrame, request.getfixturevalue(f"{dataset}_golden"))
Expand Down
Loading

0 comments on commit e75f5f5

Please sign in to comment.