delete 1 file and update 416 files

docs/adding_a_dataset.md

@@ -30,6 +30,7 @@ class SciDocsReranking(AbsTaskReranking):
         reference="https://allenai.org/data/scidocs",
         type="Reranking",
         category="s2s",
+        modalities=["text"],
         eval_splits=["test"],
         eval_langs=["eng-Latn"],
         main_score="map",
@@ -38,16 +39,13 @@ class SciDocsReranking(AbsTaskReranking):
             "revision": "d3c5e1fc0b855ab6097bf1cda04dd73947d7caab",
         }
         date=("2000-01-01", "2020-12-31"), # best guess
-        form="Written",
-        domains=["Academic", "Non-fiction"],
+        domains=["Academic", "Non-fiction", "Domains"],
         task_subtypes=["Scientific Reranking"],
         license="cc-by-4.0",
-        socioeconomic_status="high",
         annotations_creators="derived",
         dialect=[],
-        text_creation="found",
-        n_samples={"test": 19599},
-        avg_character_length={"test": 69.0},
+        sample_creation="found",
+        descriptive_stats={"n_samples": {"test": 19599}, "avg_character_length": {"test": 69.0}},
         bibtex_citation="""
 @inproceedings{cohan-etal-2020-specter,
     title = "{SPECTER}: Document-level Representation Learning using Citation-informed Transformers",
@@ -94,6 +92,7 @@ class VGClustering(AbsTaskClustering):
         reference="https://huggingface.co/datasets/navjordj/VG_summarization",
         type="Clustering",
         category="p2p",
+        modalities=["text"],
         eval_splits=["test"],
         eval_langs=["nob-Latn"],
         main_score="v_measure",
@@ -106,7 +105,6 @@ class VGClustering(AbsTaskClustering):
         domains=["Academic", "Non-fiction"],
         task_subtypes=["Scientific Reranking"],
         license="cc-by-nc",
-        socioeconomic_status="high",
         annotations_creators="derived",
         dialect=[],
         text_creation="found",

mteb/abstasks/TaskMetadata.py

@@ -135,13 +135,6 @@
 logger = logging.getLogger(__name__)
 
 
-class GeneralDescriptiveStats(BaseModel):
-    """General descriptive statistics for a dataset."""
-
-    n_samples: dict[SPLIT_NAME, int]
-    avg_character_length: dict[SPLIT_NAME, float]
-
-
 class TaskMetadata(BaseModel):
     """Metadata for a task.
 
@@ -201,7 +194,7 @@ class TaskMetadata(BaseModel):
     sample_creation: SAMPLE_CREATION_METHOD | None
     bibtex_citation: str | None
 
-    stats: dict[METRIC_NAME, Optional[dict[SPLIT_NAME, METRIC_VALUE]]]
+    descriptive_stats: dict[METRIC_NAME, Optional[dict[SPLIT_NAME, METRIC_VALUE]]]
 
     @field_validator("dataset")
     def _check_dataset_path_is_specified(cls, dataset):

mteb/tasks/BitextMining/dan/BornholmskBitextMining.py

@@ -41,7 +41,10 @@ class BornholmBitextMining(AbsTaskBitextMining):
 	file = {Available Version (via Google Scholar):/Users/au554730/Zotero/storage/FBQ73ZYN/Derczynski and Kjeldsen - 2019 - Bornholmsk natural language processing Resources .pdf:application/pdf},
 }
 """,
-        stats={"n_samples": {"test": 500}, "avg_character_length": {"test": 89.7}},
+        descriptive_stats={
+            "n_samples": {"test": 500},
+            "avg_character_length": {"test": 89.7},
+        },
     )
 
     def dataset_transform(self):

mteb/tasks/BitextMining/kat/TbilisiCityHallBitextMining.py

@@ -41,7 +41,7 @@ class TbilisiCityHallBitextMining(AbsTaskBitextMining, MultilingualTask):
         annotations_creators="derived",
         dialect=[],
         bibtex_citation="",
-        stats={
+        descriptive_stats={
             "n_samples": {_EVAL_SPLIT: 1820},
             "avg_character_length": {_EVAL_SPLIT: 78},
         },

mteb/tasks/BitextMining/multilingual/BUCCBitextMining.py

@@ -56,7 +56,10 @@ class BUCCBitextMining(AbsTaskBitextMining, MultilingualTask):
     pages = "60--67",
     abstract = "This paper presents the BUCC 2017 shared task on parallel sentence extraction from comparable corpora. It recalls the design of the datasets, presents their final construction and statistics and the methods used to evaluate system results. 13 runs were submitted to the shared task by 4 teams, covering three of the four proposed language pairs: French-English (7 runs), German-English (3 runs), and Chinese-English (3 runs). The best F-scores as measured against the gold standard were 0.84 (German-English), 0.80 (French-English), and 0.43 (Chinese-English). Because of the design of the dataset, in which not all gold parallel sentence pairs are known, these are only minimum values. We examined manually a small sample of the false negative sentence pairs for the most precise French-English runs and estimated the number of parallel sentence pairs not yet in the provided gold standard. Adding them to the gold standard leads to revised estimates for the French-English F-scores of at most +1.5pt. This suggests that the BUCC 2017 datasets provide a reasonable approximate evaluation of the parallel sentence spotting task.",
 }""",
-        stats={"n_samples": {"test": 641684}, "avg_character_length": {"test": 101.3}},
+        descriptive_stats={
+            "n_samples": {"test": 641684},
+            "avg_character_length": {"test": 101.3},
+        },
     )
 
     def dataset_transform(self):

mteb/tasks/BitextMining/multilingual/BUCCBitextMiningFast.py

@@ -56,5 +56,8 @@ class BUCCBitextMiningFast(AbsTaskBitextMining, MultilingualTask):
     pages = "60--67",
     abstract = "This paper presents the BUCC 2017 shared task on parallel sentence extraction from comparable corpora. It recalls the design of the datasets, presents their final construction and statistics and the methods used to evaluate system results. 13 runs were submitted to the shared task by 4 teams, covering three of the four proposed language pairs: French-English (7 runs), German-English (3 runs), and Chinese-English (3 runs). The best F-scores as measured against the gold standard were 0.84 (German-English), 0.80 (French-English), and 0.43 (Chinese-English). Because of the design of the dataset, in which not all gold parallel sentence pairs are known, these are only minimum values. We examined manually a small sample of the false negative sentence pairs for the most precise French-English runs and estimated the number of parallel sentence pairs not yet in the provided gold standard. Adding them to the gold standard leads to revised estimates for the French-English F-scores of at most +1.5pt. This suggests that the BUCC 2017 datasets provide a reasonable approximate evaluation of the parallel sentence spotting task.",
 }""",
-        stats={"n_samples": {"test": 641684}, "avg_character_length": {"test": 101.3}},
+        descriptive_stats={
+            "n_samples": {"test": 641684},
+            "avg_character_length": {"test": 101.3},
+        },
     )

mteb/tasks/BitextMining/multilingual/BibleNLPBitextMining.py

@@ -884,7 +884,10 @@ class BibleNLPBitextMining(AbsTaskBitextMining, MultilingualTask):
         annotations_creators="expert-annotated",
         dialect=[],
         sample_creation="created",
-        stats={"n_samples": {"train": _N}, "avg_character_length": {"train": 120}},
+        descriptive_stats={
+            "n_samples": {"train": _N},
+            "avg_character_length": {"train": 120},
+        },
         bibtex_citation="""@article{akerman2023ebible,
             title={The eBible Corpus: Data and Model Benchmarks for Bible Translation for Low-Resource Languages},
             author={Akerman, Vesa and Baines, David and Daspit, Damien and Hermjakob, Ulf and Jang, Taeho and Leong, Colin and Martin, Michael and Mathew, Joel and Robie, Jonathan and Schwarting, Marcus},

mteb/tasks/BitextMining/multilingual/DiaBLaBitextMining.py

@@ -42,7 +42,7 @@ class DiaBLaBitextMining(AbsTaskBitextMining, MultilingualTask):
         year={2019}
         }
         """,
-        stats={"n_samples": {}, "avg_character_length": {}},
+        descriptive_stats={"n_samples": {}, "avg_character_length": {}},
     )
 
     def load_data(self, **kwargs):

mteb/tasks/BitextMining/multilingual/FloresBitextMining.py

@@ -268,7 +268,10 @@ class FloresBitextMining(AbsTaskBitextMining, MultilingualTask):
         year={2022}
         }
         """,
-        stats={"n_samples": {"dev": 997, "devtest": 1012}, "avg_character_length": {}},
+        descriptive_stats={
+            "n_samples": {"dev": 997, "devtest": 1012},
+            "avg_character_length": {},
+        },
     )
 
     def load_data(self, **kwargs: Any) -> None:

mteb/tasks/BitextMining/multilingual/IN22ConvBitextMining.py

@@ -100,7 +100,10 @@ class IN22ConvBitextMining(AbsTaskBitextMining, MultilingualTask):
 url={https://openreview.net/forum?id=vfT4YuzAYA},
 note={}
 }""",
-        stats={"n_samples": {"test": 1503}, "avg_character_length": {"test": 54.3}},
+        descriptive_stats={
+            "n_samples": {"test": 1503},
+            "avg_character_length": {"test": 54.3},
+        },
     )
 
     def load_data(self, **kwargs: Any) -> None:

mteb/tasks/BitextMining/multilingual/IN22GenBitextMining.py

@@ -94,7 +94,10 @@ class IN22GenBitextMining(AbsTaskBitextMining, MultilingualTask):
 url={https://openreview.net/forum?id=vfT4YuzAYA},
 note={}
 }""",
-        stats={"n_samples": {"test": 1024}, "avg_character_length": {"test": 156.7}},
+        descriptive_stats={
+            "n_samples": {"test": 1024},
+            "avg_character_length": {"test": 156.7},
+        },
     )
 
     def load_data(self, **kwargs: Any) -> None:

mteb/tasks/BitextMining/multilingual/IWSLT2017BitextMinig.py

@@ -81,7 +81,7 @@ class IWSLT2017BitextMining(AbsTaskBitextMining, MultilingualTask):
     pages = "2--14",
 }
 """,
-        stats={
+        descriptive_stats={
             "n_samples": {"validation": 21928},
             "avg_character_length": {"validation": 95.4},
         },

mteb/tasks/BitextMining/multilingual/IndicGenBenchFloresBitextMining.py

@@ -128,7 +128,7 @@ class IndicGenBenchFloresBitextMining(AbsTaskBitextMining, MultilingualTask):
       archivePrefix={arXiv},
       primaryClass={cs.CL}
 }""",
-        stats={
+        descriptive_stats={
             "n_samples": {"validation": 997, "test": 1012},
             "avg_character_length": {"validation": 126.25, "test": 130.84},
         },

mteb/tasks/BitextMining/multilingual/LinceMTBitextMining.py

@@ -40,5 +40,8 @@ class LinceMTBitextMining(AbsTaskBitextMining, MultilingualTask):
         year={2020}
         }
         """,
-        stats={"n_samples": {"train": 8060}, "avg_character_length": {"train": 58.67}},
+        descriptive_stats={
+            "n_samples": {"train": 8060},
+            "avg_character_length": {"train": 58.67},
+        },
     )

mteb/tasks/BitextMining/multilingual/NTREXBitextMining.py

@@ -269,7 +269,7 @@ class NTREXBitextMining(AbsTaskBitextMining, MultilingualTask):
         annotations_creators="expert-annotated",
         dialect=[],
         sample_creation="human-translated and localized",
-        stats={
+        descriptive_stats={
             "n_samples": {"test": _N * len(_EVAL_LANGS)},
             "avg_character_length": {"test": 120},
         },

mteb/tasks/BitextMining/multilingual/NollySentiBitextMining.py

@@ -43,5 +43,8 @@ class NollySentiBitextMining(AbsTaskBitextMining, MultilingualTask):
             year={2023}
         }
         """,
-        stats={"n_samples": {"train": 1640}, "avg_character_length": {"train": 135.91}},
+        descriptive_stats={
+            "n_samples": {"train": 1640},
+            "avg_character_length": {"train": 135.91},
+        },
     )

mteb/tasks/BitextMining/multilingual/NorwegianCourtsBitextMining.py

@@ -34,7 +34,10 @@ class NorwegianCourtsBitextMining(AbsTaskBitextMining):
   year={2020}
 }
 """,
-        stats={"n_samples": {"test": 2050}, "avg_character_length": {"test": 1884.0}},
+        descriptive_stats={
+            "n_samples": {"test": 2050},
+            "avg_character_length": {"test": 1884.0},
+        },
     )
 
     def dataset_transform(self):

mteb/tasks/BitextMining/multilingual/NusaTranslationBitextMining.py

@@ -51,7 +51,7 @@ class NusaTranslationBitextMining(AbsTaskBitextMining, MultilingualTask):
         }
 
         """,
-        stats={
+        descriptive_stats={
             "n_samples": {"train": 50200},
             "avg_character_length": {"train": 147.01},
         },

mteb/tasks/BitextMining/multilingual/NusaXBitextMining.py

@@ -58,5 +58,8 @@ class NusaXBitextMining(AbsTaskBitextMining, MultilingualTask):
             primaryClass={cs.CL}
         }
         """,
-        stats={"n_samples": {"train": 5500}, "avg_character_length": {"train": 157.15}},
+        descriptive_stats={
+            "n_samples": {"train": 5500},
+            "avg_character_length": {"train": 157.15},
+        },
     )

mteb/tasks/BitextMining/multilingual/PhincBitextMining.py

@@ -40,5 +40,8 @@ class PhincBitextMining(AbsTaskBitextMining, MultilingualTask):
         year={2020}
         }
         """,
-        stats={"n_samples": {"train": 13738}, "avg_character_length": {"train": 75.32}},
+        descriptive_stats={
+            "n_samples": {"train": 13738},
+            "avg_character_length": {"train": 75.32},
+        },
     )

mteb/tasks/BitextMining/multilingual/RomaTalesBitextMining.py

@@ -32,7 +32,7 @@ class RomaTalesBitextMining(AbsTaskBitextMining, MultilingualTask):
         dialect=["Lovari"],
         sample_creation="created",
         bibtex_citation="",
-        stats={
+        descriptive_stats={
             "n_samples": {"test": 215},
             "avg_character_length": {"test": 316.8046511627907},
         },

mteb/tasks/BitextMining/multilingual/TatoebaBitextMining.py

@@ -152,5 +152,8 @@ class TatoebaBitextMining(AbsTaskBitextMining, MultilingualTask):
         year = {2021},
         }
         """,
-        stats={"n_samples": {"test": 2000}, "avg_character_length": {"test": 39.4}},
+        descriptive_stats={
+            "n_samples": {"test": 2000},
+            "avg_character_length": {"test": 39.4},
+        },
     )

mteb/tasks/BitextMining/srn/SRNCorpusBitextMining.py

@@ -45,7 +45,10 @@ class SRNCorpusBitextMining(AbsTaskBitextMining, MultilingualTask):
         annotations_creators="human-annotated",
         dialect=[],
         sample_creation="found",
-        stats={"n_samples": {"test": _N}, "avg_character_length": {"test": 55}},
+        descriptive_stats={
+            "n_samples": {"test": _N},
+            "avg_character_length": {"test": 55},
+        },
         bibtex_citation="""
 @article{zwennicker2022towards,
   title={Towards a general purpose machine translation system for Sranantongo},

mteb/tasks/BitextMining/vie/VieMedEVBitextMining.py

@@ -39,7 +39,7 @@ class VieMedEVBitextMining(AbsTaskBitextMining):
     booktitle = {Proceedings of the 2024 Joint International Conference on Computational Linguistics, Language Resources and Evaluation (LREC-COLING)},
     year      = {2024}
 }""",
-        stats={
+        descriptive_stats={
             "n_samples": {"test": TEST_SAMPLES},
             "avg_character_length": {"test": 139.23},
         },

mteb/tasks/Classification/ara/AJGT.py

@@ -36,5 +36,8 @@ class AJGT(AbsTaskClassification):
   organization={Springer}
 }
 """,
-        stats={"n_samples": {"train": 1800}, "avg_character_length": {"train": 46.81}},
+        descriptive_stats={
+            "n_samples": {"train": 1800},
+            "avg_character_length": {"train": 46.81},
+        },
     )

mteb/tasks/Classification/ara/HotelReviewSentimentClassification.py

@@ -39,7 +39,7 @@ class HotelReviewSentimentClassification(AbsTaskClassification):
   publisher={Springer}
 }
 """,
-        stats={
+        descriptive_stats={
             "n_samples": {"train": N_SAMPLES},
             "avg_character_length": {"train": 137.2},
         },

mteb/tasks/Classification/ara/OnlineStoreReviewSentimentClassification.py

@@ -29,7 +29,7 @@ class OnlineStoreReviewSentimentClassification(AbsTaskClassification):
         dialect=["ara-Arab-SA"],
         sample_creation="found",
         bibtex_citation="",
-        stats={
+        descriptive_stats={
             "n_samples": {"train": N_SAMPLES},
             "avg_character_length": {"train": 137.2},
         },

mteb/tasks/Classification/ara/RestaurantReviewSentimentClassification.py

@@ -38,7 +38,7 @@ class RestaurantReviewSentimentClassification(AbsTaskClassification):
   organization={Springer}
 }
 """,
-        stats={
+        descriptive_stats={
             "n_samples": {"train": N_SAMPLES},
             "avg_character_length": {"train": 231.4},
         },

mteb/tasks/Classification/ara/TweetEmotionClassification.py

@@ -39,7 +39,7 @@ class TweetEmotionClassification(AbsTaskClassification):
   organization={Springer}
 }
 """,
-        stats={
+        descriptive_stats={
             "n_samples": {"train": N_SAMPLES},
             "avg_character_length": {"train": 78.8},
         },

mteb/tasks/Classification/ara/TweetSarcasmClassification.py

@@ -48,7 +48,10 @@ class TweetSarcasmClassification(AbsTaskClassification):
     ISBN = "979-10-95546-51-1",
 }
 """,
-        stats={"n_samples": {"test": 2110}, "avg_character_length": {"test": 102.1}},
+        descriptive_stats={
+            "n_samples": {"test": 2110},
+            "avg_character_length": {"test": 102.1},
+        },
     )
 
     def dataset_transform(self):

mteb/tasks/Classification/ben/BengaliDocumentClassification.py

@@ -42,7 +42,10 @@ class BengaliDocumentClassification(AbsTaskClassification):
             pages = "52--67"
         }
         """,
-        stats={"n_samples": {"test": 2048}, "avg_character_length": {"test": 1658.1}},
+        descriptive_stats={
+            "n_samples": {"test": 2048},
+            "avg_character_length": {"test": 1658.1},
+        },
     )
 
     def dataset_transform(self) -> None: