Skip to content

Commit

Permalink
added GoogleIndicGenBench Flores , Crosssum , Xqora , Xquad - Adithya…
Browse files Browse the repository at this point in the history 
… S K
  • Loading branch information
@adithya-s-k
adithya-s-k committed Jun 4, 2024
1 parent f9fe52f commit 385816f
Show file tree
Hide file tree
Showing 3 changed files with 66 additions and 1 deletion.
2 changes: 1 addition & 1 deletion pyproject.toml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@ where = ["src"]

[project]
name = "indic_eval"
version = "0.3.0.dev0"
version = "0.1.0"
authors = [
{ name="Adithya S Kolavi", email="[email protected]" }
]
Expand Down
32 changes: 32 additions & 0 deletions src/indic_eval/tasks/indic_tasks_table.jsonl
View file
Original file line number Diff line number Diff line change
@@ -1,3 +1,35 @@
{"name":"Flores:hindi","suite":["indiceval","leaderboard","flores"],"prompt_function":"flores_indic","hf_repo":"Cognitive-Lab\/GoogleIndicGenBench_flores_in","hf_subset":"hi","hf_avail_splits":["test","dev"],"evaluation_splits":["test"],"few_shots_split":"dev","few_shots_select":"random_sampling_from_train","generation_size":1,"metric":["loglikelihood_acc","loglikelihood_acc_norm_nospace"],"stop_sequence":["\n"],"output_regex":null,"frozen":false, "trust_dataset": true}
{"name":"Flores:kannada","suite":["indiceval","leaderboard","flores"],"prompt_function":"flores_indic","hf_repo":"Cognitive-Lab\/GoogleIndicGenBench_flores_in","hf_subset":"kn","hf_avail_splits":["test","dev"],"evaluation_splits":["test"],"few_shots_split":"dev","few_shots_select":"random_sampling_from_train","generation_size":1,"metric":["loglikelihood_acc","loglikelihood_acc_norm_nospace"],"stop_sequence":["\n"],"output_regex":null,"frozen":false, "trust_dataset": true}
{"name":"Flores:tamil","suite":["indiceval","leaderboard","flores"],"prompt_function":"flores_indic","hf_repo":"Cognitive-Lab\/GoogleIndicGenBench_flores_in","hf_subset":"ta","hf_avail_splits":["test","dev"],"evaluation_splits":["test"],"few_shots_split":"dev","few_shots_select":"random_sampling_from_train","generation_size":1,"metric":["loglikelihood_acc","loglikelihood_acc_norm_nospace"],"stop_sequence":["\n"],"output_regex":null,"frozen":false, "trust_dataset": true}
{"name":"Flores:telugu","suite":["indiceval","leaderboard","flores"],"prompt_function":"flores_indic","hf_repo":"Cognitive-Lab\/GoogleIndicGenBench_flores_in","hf_subset":"te","hf_avail_splits":["test","dev"],"evaluation_splits":["test"],"few_shots_split":"dev","few_shots_select":"random_sampling_from_train","generation_size":1,"metric":["loglikelihood_acc","loglikelihood_acc_norm_nospace"],"stop_sequence":["\n"],"output_regex":null,"frozen":false, "trust_dataset": true}
{"name":"Flores:gujarati","suite":["indiceval","leaderboard","flores"],"prompt_function":"flores_indic","hf_repo":"Cognitive-Lab\/GoogleIndicGenBench_flores_in","hf_subset":"gu","hf_avail_splits":["test","dev"],"evaluation_splits":["test"],"few_shots_split":"dev","few_shots_select":"random_sampling_from_train","generation_size":1,"metric":["loglikelihood_acc","loglikelihood_acc_norm_nospace"],"stop_sequence":["\n"],"output_regex":null,"frozen":false, "trust_dataset": true}
{"name":"Flores:marathi","suite":["indiceval","leaderboard","flores"],"prompt_function":"flores_indic","hf_repo":"Cognitive-Lab\/GoogleIndicGenBench_flores_in","hf_subset":"mr","hf_avail_splits":["test","dev"],"evaluation_splits":["test"],"few_shots_split":"dev","few_shots_select":"random_sampling_from_train","generation_size":1,"metric":["loglikelihood_acc","loglikelihood_acc_norm_nospace"],"stop_sequence":["\n"],"output_regex":null,"frozen":false, "trust_dataset": true}
{"name":"Flores:malayalam","suite":["indiceval","leaderboard","flores"],"prompt_function":"flores_indic","hf_repo":"Cognitive-Lab\/GoogleIndicGenBench_flores_in","hf_subset":"ml","hf_avail_splits":["test","dev"],"evaluation_splits":["test"],"few_shots_split":"dev","few_shots_select":"random_sampling_from_train","generation_size":1,"metric":["loglikelihood_acc","loglikelihood_acc_norm_nospace"],"stop_sequence":["\n"],"output_regex":null,"frozen":false, "trust_dataset": true}

{"name":"Crosssum:hindi","suite":["indiceval","leaderboard","crosssum"],"prompt_function":"crosssum_indic","hf_repo":"Cognitive-Lab\/GoogleIndicGenBench_crosssum_in","hf_subset":"hi","hf_avail_splits":["test","dev"],"evaluation_splits":["test"],"few_shots_split":"dev","few_shots_select":"random_sampling_from_train","generation_size":1,"metric":["loglikelihood_acc","loglikelihood_acc_norm_nospace"],"stop_sequence":["\n"],"output_regex":null,"frozen":false, "trust_dataset": true}
{"name":"Crosssum:kannada","suite":["indiceval","leaderboard","crosssum"],"prompt_function":"crosssum_indic","hf_repo":"Cognitive-Lab\/GoogleIndicGenBench_crosssum_in","hf_subset":"kn","hf_avail_splits":["test","dev"],"evaluation_splits":["test"],"few_shots_split":"dev","few_shots_select":"random_sampling_from_train","generation_size":1,"metric":["loglikelihood_acc","loglikelihood_acc_norm_nospace"],"stop_sequence":["\n"],"output_regex":null,"frozen":false, "trust_dataset": true}
{"name":"Crosssum:tamil","suite":["indiceval","leaderboard","crosssum"],"prompt_function":"crosssum_indic","hf_repo":"Cognitive-Lab\/GoogleIndicGenBench_crosssum_in","hf_subset":"ta","hf_avail_splits":["test","dev"],"evaluation_splits":["test"],"few_shots_split":"dev","few_shots_select":"random_sampling_from_train","generation_size":1,"metric":["loglikelihood_acc","loglikelihood_acc_norm_nospace"],"stop_sequence":["\n"],"output_regex":null,"frozen":false, "trust_dataset": true}
{"name":"Crosssum:telugu","suite":["indiceval","leaderboard","crosssum"],"prompt_function":"crosssum_indic","hf_repo":"Cognitive-Lab\/GoogleIndicGenBench_crosssum_in","hf_subset":"te","hf_avail_splits":["test","dev"],"evaluation_splits":["test"],"few_shots_split":"dev","few_shots_select":"random_sampling_from_train","generation_size":1,"metric":["loglikelihood_acc","loglikelihood_acc_norm_nospace"],"stop_sequence":["\n"],"output_regex":null,"frozen":false, "trust_dataset": true}
{"name":"Crosssum:gujarati","suite":["indiceval","leaderboard","crosssum"],"prompt_function":"crosssum_indic","hf_repo":"Cognitive-Lab\/GoogleIndicGenBench_crosssum_in","hf_subset":"gu","hf_avail_splits":["test","dev"],"evaluation_splits":["test"],"few_shots_split":"dev","few_shots_select":"random_sampling_from_train","generation_size":1,"metric":["loglikelihood_acc","loglikelihood_acc_norm_nospace"],"stop_sequence":["\n"],"output_regex":null,"frozen":false, "trust_dataset": true}
{"name":"Crosssum:marathi","suite":["indiceval","leaderboard","crosssum"],"prompt_function":"crosssum_indic","hf_repo":"Cognitive-Lab\/GoogleIndicGenBench_crosssum_in","hf_subset":"mr","hf_avail_splits":["test","dev"],"evaluation_splits":["test"],"few_shots_split":"dev","few_shots_select":"random_sampling_from_train","generation_size":1,"metric":["loglikelihood_acc","loglikelihood_acc_norm_nospace"],"stop_sequence":["\n"],"output_regex":null,"frozen":false, "trust_dataset": true}
{"name":"Crosssum:malayalam","suite":["indiceval","leaderboard","crosssum"],"prompt_function":"crosssum_indic","hf_repo":"Cognitive-Lab\/GoogleIndicGenBench_crosssum_in","hf_subset":"ml","hf_avail_splits":["test","dev"],"evaluation_splits":["test"],"few_shots_split":"dev","few_shots_select":"random_sampling_from_train","generation_size":1,"metric":["loglikelihood_acc","loglikelihood_acc_norm_nospace"],"stop_sequence":["\n"],"output_regex":null,"frozen":false, "trust_dataset": true}

{"name":"Xorqa:hindi","suite":["indiceval","leaderboard","xorqa"],"prompt_function":"xorqa_indic","hf_repo":"Cognitive-Lab\/GoogleIndicGenBench_xorqa_in","hf_subset":"hi","hf_avail_splits":["test","dev"],"evaluation_splits":["test"],"few_shots_split":"dev","few_shots_select":"random_sampling_from_train","generation_size":1,"metric":["loglikelihood_acc","loglikelihood_acc_norm_nospace"],"stop_sequence":["\n"],"output_regex":null,"frozen":false, "trust_dataset": true}
{"name":"Xorqa:kannada","suite":["indiceval","leaderboard","xorqa"],"prompt_function":"xorqa_indic","hf_repo":"Cognitive-Lab\/GoogleIndicGenBench_xorqa_in","hf_subset":"kn","hf_avail_splits":["test","dev"],"evaluation_splits":["test"],"few_shots_split":"dev","few_shots_select":"random_sampling_from_train","generation_size":1,"metric":["loglikelihood_acc","loglikelihood_acc_norm_nospace"],"stop_sequence":["\n"],"output_regex":null,"frozen":false, "trust_dataset": true}
{"name":"Xorqa:tamil","suite":["indiceval","leaderboard","xorqa"],"prompt_function":"xorqa_indic","hf_repo":"Cognitive-Lab\/GoogleIndicGenBench_xorqa_in","hf_subset":"ta","hf_avail_splits":["test","dev"],"evaluation_splits":["test"],"few_shots_split":"dev","few_shots_select":"random_sampling_from_train","generation_size":1,"metric":["loglikelihood_acc","loglikelihood_acc_norm_nospace"],"stop_sequence":["\n"],"output_regex":null,"frozen":false, "trust_dataset": true}
{"name":"Xorqa:telugu","suite":["indiceval","leaderboard","xorqa"],"prompt_function":"xorqa_indic","hf_repo":"Cognitive-Lab\/GoogleIndicGenBench_xorqa_in","hf_subset":"te","hf_avail_splits":["test","dev"],"evaluation_splits":["test"],"few_shots_split":"dev","few_shots_select":"random_sampling_from_train","generation_size":1,"metric":["loglikelihood_acc","loglikelihood_acc_norm_nospace"],"stop_sequence":["\n"],"output_regex":null,"frozen":false, "trust_dataset": true}
{"name":"Xorqa:gujarati","suite":["indiceval","leaderboard","xorqa"],"prompt_function":"xorqa_indic","hf_repo":"Cognitive-Lab\/GoogleIndicGenBench_xorqa_in","hf_subset":"gu","hf_avail_splits":["test","dev"],"evaluation_splits":["test"],"few_shots_split":"dev","few_shots_select":"random_sampling_from_train","generation_size":1,"metric":["loglikelihood_acc","loglikelihood_acc_norm_nospace"],"stop_sequence":["\n"],"output_regex":null,"frozen":false, "trust_dataset": true}
{"name":"Xorqa:marathi","suite":["indiceval","leaderboard","xorqa"],"prompt_function":"xorqa_indic","hf_repo":"Cognitive-Lab\/GoogleIndicGenBench_xorqa_in","hf_subset":"mr","hf_avail_splits":["test","dev"],"evaluation_splits":["test"],"few_shots_split":"dev","few_shots_select":"random_sampling_from_train","generation_size":1,"metric":["loglikelihood_acc","loglikelihood_acc_norm_nospace"],"stop_sequence":["\n"],"output_regex":null,"frozen":false, "trust_dataset": true}
{"name":"Xorqa:malayalam","suite":["indiceval","leaderboard","xorqa"],"prompt_function":"xorqa_indic","hf_repo":"Cognitive-Lab\/GoogleIndicGenBench_xorqa_in","hf_subset":"ml","hf_avail_splits":["test","dev"],"evaluation_splits":["test"],"few_shots_split":"dev","few_shots_select":"random_sampling_from_train","generation_size":1,"metric":["loglikelihood_acc","loglikelihood_acc_norm_nospace"],"stop_sequence":["\n"],"output_regex":null,"frozen":false, "trust_dataset": true}

{"name":"Xquad:hindi","suite":["indiceval","leaderboard","xquad"],"prompt_function":"xquad_indic","hf_repo":"Cognitive-Lab\/GoogleIndicGenBench_xquad_in","hf_subset":"hi","hf_avail_splits":["test","dev"],"evaluation_splits":["test"],"few_shots_split":"dev","few_shots_select":"random_sampling_from_train","generation_size":1,"metric":["loglikelihood_acc","loglikelihood_acc_norm_nospace"],"stop_sequence":["\n"],"output_regex":null,"frozen":false, "trust_dataset": true}
{"name":"Xquad:kannada","suite":["indiceval","leaderboard","xquad"],"prompt_function":"xquad_indic","hf_repo":"Cognitive-Lab\/GoogleIndicGenBench_xquad_in","hf_subset":"kn","hf_avail_splits":["test","dev"],"evaluation_splits":["test"],"few_shots_split":"dev","few_shots_select":"random_sampling_from_train","generation_size":1,"metric":["loglikelihood_acc","loglikelihood_acc_norm_nospace"],"stop_sequence":["\n"],"output_regex":null,"frozen":false, "trust_dataset": true}
{"name":"Xquad:tamil","suite":["indiceval","leaderboard","xquad"],"prompt_function":"xquad_indic","hf_repo":"Cognitive-Lab\/GoogleIndicGenBench_xquad_in","hf_subset":"ta","hf_avail_splits":["test","dev"],"evaluation_splits":["test"],"few_shots_split":"dev","few_shots_select":"random_sampling_from_train","generation_size":1,"metric":["loglikelihood_acc","loglikelihood_acc_norm_nospace"],"stop_sequence":["\n"],"output_regex":null,"frozen":false, "trust_dataset": true}
{"name":"Xquad:telugu","suite":["indiceval","leaderboard","xquad"],"prompt_function":"xquad_indic","hf_repo":"Cognitive-Lab\/GoogleIndicGenBench_xquad_in","hf_subset":"te","hf_avail_splits":["test","dev"],"evaluation_splits":["test"],"few_shots_split":"dev","few_shots_select":"random_sampling_from_train","generation_size":1,"metric":["loglikelihood_acc","loglikelihood_acc_norm_nospace"],"stop_sequence":["\n"],"output_regex":null,"frozen":false, "trust_dataset": true}
{"name":"Xquad:gujarati","suite":["indiceval","leaderboard","xquad"],"prompt_function":"xquad_indic","hf_repo":"Cognitive-Lab\/GoogleIndicGenBench_xquad_in","hf_subset":"gu","hf_avail_splits":["test","dev"],"evaluation_splits":["test"],"few_shots_split":"dev","few_shots_select":"random_sampling_from_train","generation_size":1,"metric":["loglikelihood_acc","loglikelihood_acc_norm_nospace"],"stop_sequence":["\n"],"output_regex":null,"frozen":false, "trust_dataset": true}
{"name":"Xquad:marathi","suite":["indiceval","leaderboard","xquad"],"prompt_function":"xquad_indic","hf_repo":"Cognitive-Lab\/GoogleIndicGenBench_xquad_in","hf_subset":"mr","hf_avail_splits":["test","dev"],"evaluation_splits":["test"],"few_shots_split":"dev","few_shots_select":"random_sampling_from_train","generation_size":1,"metric":["loglikelihood_acc","loglikelihood_acc_norm_nospace"],"stop_sequence":["\n"],"output_regex":null,"frozen":false, "trust_dataset": true}
{"name":"Xquad:malayalam","suite":["indiceval","leaderboard","xquad"],"prompt_function":"xquad_indic","hf_repo":"Cognitive-Lab\/GoogleIndicGenBench_xquad_in","hf_subset":"ml","hf_avail_splits":["test","dev"],"evaluation_splits":["test"],"few_shots_split":"dev","few_shots_select":"random_sampling_from_train","generation_size":1,"metric":["loglikelihood_acc","loglikelihood_acc_norm_nospace"],"stop_sequence":["\n"],"output_regex":null,"frozen":false, "trust_dataset": true}

{"name":"ARC-Easy:english","suite":["indiceval","leaderboard","arc"],"prompt_function":"arc","hf_repo":"ai2_arc","hf_subset":"ARC-Easy","hf_avail_splits":["train","validation","test"],"evaluation_splits":["test"],"few_shots_split":null,"few_shots_select":"random_sampling_from_train","generation_size":1,"metric":["loglikelihood_acc","loglikelihood_acc_norm_nospace"],"stop_sequence":["\n"],"output_regex":null,"frozen":false, "trust_dataset": true}
{"name":"ARC-Easy:hindi","suite":["indiceval","leaderboard","arc"],"prompt_function":"arc_indic","hf_repo":"Cognitive-Lab\/Indic-ARC-Easy","hf_subset":"hi","hf_avail_splits":["train","validation","test"],"evaluation_splits":["test"],"few_shots_split":null,"few_shots_select":"random_sampling_from_train","generation_size":1,"metric":["loglikelihood_acc","loglikelihood_acc_norm_nospace"],"stop_sequence":["\n"],"output_regex":null,"frozen":false, "trust_dataset": true}
{"name":"ARC-Easy:kannada","suite":["indiceval","leaderboard","arc"],"prompt_function":"arc_indic","hf_repo":"Cognitive-Lab\/Indic-ARC-Easy","hf_subset":"kn","hf_avail_splits":["train","validation","test"],"evaluation_splits":["test"],"few_shots_split":null,"few_shots_select":"random_sampling_from_train","generation_size":1,"metric":["loglikelihood_acc","loglikelihood_acc_norm_nospace"],"stop_sequence":["\n"],"output_regex":null,"frozen":false, "trust_dataset": true}
Expand Down
Loading

0 comments on commit 385816f

Please sign in to comment.