diff --git a/src/indic_eval/tasks/indic_tasks_table.jsonl b/src/indic_eval/tasks/indic_tasks_table.jsonl index 2d32804..8fea6eb 100644 --- a/src/indic_eval/tasks/indic_tasks_table.jsonl +++ b/src/indic_eval/tasks/indic_tasks_table.jsonl @@ -1,10 +1,18 @@ -{"name":"Flores:hindi","suite":["indiceval","leaderboard","flores"],"prompt_function":"flores_indic","hf_repo":"Cognitive-Lab\/GoogleIndicGenBench_flores_in","hf_subset":"hi","hf_avail_splits":["test","dev"],"evaluation_splits":["test"],"few_shots_split":"dev","few_shots_select":"random_sampling_from_train","generation_size":1,"metric":["loglikelihood_acc","loglikelihood_acc_norm_nospace"],"stop_sequence":["\n"],"output_regex":null,"frozen":false, "trust_dataset": true} -{"name":"Flores:kannada","suite":["indiceval","leaderboard","flores"],"prompt_function":"flores_indic","hf_repo":"Cognitive-Lab\/GoogleIndicGenBench_flores_in","hf_subset":"kn","hf_avail_splits":["test","dev"],"evaluation_splits":["test"],"few_shots_split":"dev","few_shots_select":"random_sampling_from_train","generation_size":1,"metric":["loglikelihood_acc","loglikelihood_acc_norm_nospace"],"stop_sequence":["\n"],"output_regex":null,"frozen":false, "trust_dataset": true} -{"name":"Flores:tamil","suite":["indiceval","leaderboard","flores"],"prompt_function":"flores_indic","hf_repo":"Cognitive-Lab\/GoogleIndicGenBench_flores_in","hf_subset":"ta","hf_avail_splits":["test","dev"],"evaluation_splits":["test"],"few_shots_split":"dev","few_shots_select":"random_sampling_from_train","generation_size":1,"metric":["loglikelihood_acc","loglikelihood_acc_norm_nospace"],"stop_sequence":["\n"],"output_regex":null,"frozen":false, "trust_dataset": true} -{"name":"Flores:telugu","suite":["indiceval","leaderboard","flores"],"prompt_function":"flores_indic","hf_repo":"Cognitive-Lab\/GoogleIndicGenBench_flores_in","hf_subset":"te","hf_avail_splits":["test","dev"],"evaluation_splits":["test"],"few_shots_split":"dev","few_shots_select":"random_sampling_from_train","generation_size":1,"metric":["loglikelihood_acc","loglikelihood_acc_norm_nospace"],"stop_sequence":["\n"],"output_regex":null,"frozen":false, "trust_dataset": true} -{"name":"Flores:gujarati","suite":["indiceval","leaderboard","flores"],"prompt_function":"flores_indic","hf_repo":"Cognitive-Lab\/GoogleIndicGenBench_flores_in","hf_subset":"gu","hf_avail_splits":["test","dev"],"evaluation_splits":["test"],"few_shots_split":"dev","few_shots_select":"random_sampling_from_train","generation_size":1,"metric":["loglikelihood_acc","loglikelihood_acc_norm_nospace"],"stop_sequence":["\n"],"output_regex":null,"frozen":false, "trust_dataset": true} -{"name":"Flores:marathi","suite":["indiceval","leaderboard","flores"],"prompt_function":"flores_indic","hf_repo":"Cognitive-Lab\/GoogleIndicGenBench_flores_in","hf_subset":"mr","hf_avail_splits":["test","dev"],"evaluation_splits":["test"],"few_shots_split":"dev","few_shots_select":"random_sampling_from_train","generation_size":1,"metric":["loglikelihood_acc","loglikelihood_acc_norm_nospace"],"stop_sequence":["\n"],"output_regex":null,"frozen":false, "trust_dataset": true} -{"name":"Flores:malayalam","suite":["indiceval","leaderboard","flores"],"prompt_function":"flores_indic","hf_repo":"Cognitive-Lab\/GoogleIndicGenBench_flores_in","hf_subset":"ml","hf_avail_splits":["test","dev"],"evaluation_splits":["test"],"few_shots_split":"dev","few_shots_select":"random_sampling_from_train","generation_size":1,"metric":["loglikelihood_acc","loglikelihood_acc_norm_nospace"],"stop_sequence":["\n"],"output_regex":null,"frozen":false, "trust_dataset": true} +{"name":"Flores_enxx:hindi","suite":["indiceval","leaderboard","flores_enxx"],"prompt_function":"flores_enxx_indic","hf_repo":"Cognitive-Lab\/GoogleIndicGenBench_flores_enxx_in","hf_subset":"hi","hf_avail_splits":["test","dev"],"evaluation_splits":["test"],"few_shots_split":"dev","few_shots_select":"random_sampling_from_train","generation_size":1,"metric":["loglikelihood_acc","loglikelihood_acc_norm_nospace"],"stop_sequence":["\n"],"output_regex":null,"frozen":false, "trust_dataset": true} +{"name":"Flores_enxx:kannada","suite":["indiceval","leaderboard","flores_enxx"],"prompt_function":"flores_enxx_indic","hf_repo":"Cognitive-Lab\/GoogleIndicGenBench_flores_enxx_in","hf_subset":"kn","hf_avail_splits":["test","dev"],"evaluation_splits":["test"],"few_shots_split":"dev","few_shots_select":"random_sampling_from_train","generation_size":1,"metric":["loglikelihood_acc","loglikelihood_acc_norm_nospace"],"stop_sequence":["\n"],"output_regex":null,"frozen":false, "trust_dataset": true} +{"name":"Flores_enxx:tamil","suite":["indiceval","leaderboard","flores_enxx"],"prompt_function":"flores_enxx_indic","hf_repo":"Cognitive-Lab\/GoogleIndicGenBench_flores_enxx_in","hf_subset":"ta","hf_avail_splits":["test","dev"],"evaluation_splits":["test"],"few_shots_split":"dev","few_shots_select":"random_sampling_from_train","generation_size":1,"metric":["loglikelihood_acc","loglikelihood_acc_norm_nospace"],"stop_sequence":["\n"],"output_regex":null,"frozen":false, "trust_dataset": true} +{"name":"Flores_enxx:telugu","suite":["indiceval","leaderboard","flores_enxx"],"prompt_function":"flores_enxx_indic","hf_repo":"Cognitive-Lab\/GoogleIndicGenBench_flores_enxx_in","hf_subset":"te","hf_avail_splits":["test","dev"],"evaluation_splits":["test"],"few_shots_split":"dev","few_shots_select":"random_sampling_from_train","generation_size":1,"metric":["loglikelihood_acc","loglikelihood_acc_norm_nospace"],"stop_sequence":["\n"],"output_regex":null,"frozen":false, "trust_dataset": true} +{"name":"Flores_enxx:gujarati","suite":["indiceval","leaderboard","flores_enxx"],"prompt_function":"flores_enxx_indic","hf_repo":"Cognitive-Lab\/GoogleIndicGenBench_flores_enxx_in","hf_subset":"gu","hf_avail_splits":["test","dev"],"evaluation_splits":["test"],"few_shots_split":"dev","few_shots_select":"random_sampling_from_train","generation_size":1,"metric":["loglikelihood_acc","loglikelihood_acc_norm_nospace"],"stop_sequence":["\n"],"output_regex":null,"frozen":false, "trust_dataset": true} +{"name":"Flores_enxx:marathi","suite":["indiceval","leaderboard","flores_enxx"],"prompt_function":"flores_enxx_indic","hf_repo":"Cognitive-Lab\/GoogleIndicGenBench_flores_enxx_in","hf_subset":"mr","hf_avail_splits":["test","dev"],"evaluation_splits":["test"],"few_shots_split":"dev","few_shots_select":"random_sampling_from_train","generation_size":1,"metric":["loglikelihood_acc","loglikelihood_acc_norm_nospace"],"stop_sequence":["\n"],"output_regex":null,"frozen":false, "trust_dataset": true} +{"name":"Flores_enxx:malayalam","suite":["indiceval","leaderboard","flores_enxx"],"prompt_function":"flores_enxx_indic","hf_repo":"Cognitive-Lab\/GoogleIndicGenBench_flores_enxx_in","hf_subset":"ml","hf_avail_splits":["test","dev"],"evaluation_splits":["test"],"few_shots_split":"dev","few_shots_select":"random_sampling_from_train","generation_size":1,"metric":["loglikelihood_acc","loglikelihood_acc_norm_nospace"],"stop_sequence":["\n"],"output_regex":null,"frozen":false, "trust_dataset": true} + +{"name":"Flores_xxen:hindi","suite":["indiceval","leaderboard","flores_xxen"],"prompt_function":"flores_xxen_indic","hf_repo":"Cognitive-Lab\/GoogleIndicGenBench_flores_xxen_in","hf_subset":"hi","hf_avail_splits":["test","dev"],"evaluation_splits":["test"],"few_shots_split":"dev","few_shots_select":"random_sampling_from_train","generation_size":1,"metric":["loglikelihood_acc","loglikelihood_acc_norm_nospace"],"stop_sequence":["\n"],"output_regex":null,"frozen":false, "trust_dataset": true} +{"name":"Flores_xxen:kannada","suite":["indiceval","leaderboard","flores_xxen"],"prompt_function":"flores_xxen_indic","hf_repo":"Cognitive-Lab\/GoogleIndicGenBench_flores_xxen_in","hf_subset":"kn","hf_avail_splits":["test","dev"],"evaluation_splits":["test"],"few_shots_split":"dev","few_shots_select":"random_sampling_from_train","generation_size":1,"metric":["loglikelihood_acc","loglikelihood_acc_norm_nospace"],"stop_sequence":["\n"],"output_regex":null,"frozen":false, "trust_dataset": true} +{"name":"Flores_xxen:tamil","suite":["indiceval","leaderboard","flores_xxen"],"prompt_function":"flores_xxen_indic","hf_repo":"Cognitive-Lab\/GoogleIndicGenBench_flores_xxen_in","hf_subset":"ta","hf_avail_splits":["test","dev"],"evaluation_splits":["test"],"few_shots_split":"dev","few_shots_select":"random_sampling_from_train","generation_size":1,"metric":["loglikelihood_acc","loglikelihood_acc_norm_nospace"],"stop_sequence":["\n"],"output_regex":null,"frozen":false, "trust_dataset": true} +{"name":"Flores_xxen:telugu","suite":["indiceval","leaderboard","flores_xxen"],"prompt_function":"flores_xxen_indic","hf_repo":"Cognitive-Lab\/GoogleIndicGenBench_flores_xxen_in","hf_subset":"te","hf_avail_splits":["test","dev"],"evaluation_splits":["test"],"few_shots_split":"dev","few_shots_select":"random_sampling_from_train","generation_size":1,"metric":["loglikelihood_acc","loglikelihood_acc_norm_nospace"],"stop_sequence":["\n"],"output_regex":null,"frozen":false, "trust_dataset": true} +{"name":"Flores_xxen:gujarati","suite":["indiceval","leaderboard","flores_xxen"],"prompt_function":"flores_xxen_indic","hf_repo":"Cognitive-Lab\/GoogleIndicGenBench_flores_xxen_in","hf_subset":"gu","hf_avail_splits":["test","dev"],"evaluation_splits":["test"],"few_shots_split":"dev","few_shots_select":"random_sampling_from_train","generation_size":1,"metric":["loglikelihood_acc","loglikelihood_acc_norm_nospace"],"stop_sequence":["\n"],"output_regex":null,"frozen":false, "trust_dataset": true} +{"name":"Flores_xxen:marathi","suite":["indiceval","leaderboard","flores_xxen"],"prompt_function":"flores_xxen_indic","hf_repo":"Cognitive-Lab\/GoogleIndicGenBench_flores_xxen_in","hf_subset":"mr","hf_avail_splits":["test","dev"],"evaluation_splits":["test"],"few_shots_split":"dev","few_shots_select":"random_sampling_from_train","generation_size":1,"metric":["loglikelihood_acc","loglikelihood_acc_norm_nospace"],"stop_sequence":["\n"],"output_regex":null,"frozen":false, "trust_dataset": true} +{"name":"Flores_xxen:malayalam","suite":["indiceval","leaderboard","flores_xxen"],"prompt_function":"flores_xxen_indic","hf_repo":"Cognitive-Lab\/GoogleIndicGenBench_flores_xxen_in","hf_subset":"ml","hf_avail_splits":["test","dev"],"evaluation_splits":["test"],"few_shots_split":"dev","few_shots_select":"random_sampling_from_train","generation_size":1,"metric":["loglikelihood_acc","loglikelihood_acc_norm_nospace"],"stop_sequence":["\n"],"output_regex":null,"frozen":false, "trust_dataset": true} {"name":"Crosssum:hindi","suite":["indiceval","leaderboard","crosssum"],"prompt_function":"crosssum_indic","hf_repo":"Cognitive-Lab\/GoogleIndicGenBench_crosssum_in","hf_subset":"hi","hf_avail_splits":["test","dev"],"evaluation_splits":["test"],"few_shots_split":"dev","few_shots_select":"random_sampling_from_train","generation_size":1,"metric":["loglikelihood_acc","loglikelihood_acc_norm_nospace"],"stop_sequence":["\n"],"output_regex":null,"frozen":false, "trust_dataset": true} {"name":"Crosssum:kannada","suite":["indiceval","leaderboard","crosssum"],"prompt_function":"crosssum_indic","hf_repo":"Cognitive-Lab\/GoogleIndicGenBench_crosssum_in","hf_subset":"kn","hf_avail_splits":["test","dev"],"evaluation_splits":["test"],"few_shots_split":"dev","few_shots_select":"random_sampling_from_train","generation_size":1,"metric":["loglikelihood_acc","loglikelihood_acc_norm_nospace"],"stop_sequence":["\n"],"output_regex":null,"frozen":false, "trust_dataset": true} diff --git a/src/indic_eval/tasks/tasks_table.jsonl b/src/indic_eval/tasks/tasks_table.jsonl index 3ae5147..a47cf3e 100644 --- a/src/indic_eval/tasks/tasks_table.jsonl +++ b/src/indic_eval/tasks/tasks_table.jsonl @@ -1,10 +1,18 @@ -{"name":"Flores:hindi","suite":["indiceval","leaderboard","flores"],"prompt_function":"flores_indic","hf_repo":"Cognitive-Lab\/GoogleIndicGenBench_flores_in","hf_subset":"hi","hf_avail_splits":["test","dev"],"evaluation_splits":["test"],"few_shots_split":"dev","few_shots_select":"random_sampling_from_train","generation_size":1,"metric":["loglikelihood_acc","loglikelihood_acc_norm_nospace"],"stop_sequence":["\n"],"output_regex":null,"frozen":false, "trust_dataset": true} -{"name":"Flores:kannada","suite":["indiceval","leaderboard","flores"],"prompt_function":"flores_indic","hf_repo":"Cognitive-Lab\/GoogleIndicGenBench_flores_in","hf_subset":"kn","hf_avail_splits":["test","dev"],"evaluation_splits":["test"],"few_shots_split":"dev","few_shots_select":"random_sampling_from_train","generation_size":1,"metric":["loglikelihood_acc","loglikelihood_acc_norm_nospace"],"stop_sequence":["\n"],"output_regex":null,"frozen":false, "trust_dataset": true} -{"name":"Flores:tamil","suite":["indiceval","leaderboard","flores"],"prompt_function":"flores_indic","hf_repo":"Cognitive-Lab\/GoogleIndicGenBench_flores_in","hf_subset":"ta","hf_avail_splits":["test","dev"],"evaluation_splits":["test"],"few_shots_split":"dev","few_shots_select":"random_sampling_from_train","generation_size":1,"metric":["loglikelihood_acc","loglikelihood_acc_norm_nospace"],"stop_sequence":["\n"],"output_regex":null,"frozen":false, "trust_dataset": true} -{"name":"Flores:telugu","suite":["indiceval","leaderboard","flores"],"prompt_function":"flores_indic","hf_repo":"Cognitive-Lab\/GoogleIndicGenBench_flores_in","hf_subset":"te","hf_avail_splits":["test","dev"],"evaluation_splits":["test"],"few_shots_split":"dev","few_shots_select":"random_sampling_from_train","generation_size":1,"metric":["loglikelihood_acc","loglikelihood_acc_norm_nospace"],"stop_sequence":["\n"],"output_regex":null,"frozen":false, "trust_dataset": true} -{"name":"Flores:gujarati","suite":["indiceval","leaderboard","flores"],"prompt_function":"flores_indic","hf_repo":"Cognitive-Lab\/GoogleIndicGenBench_flores_in","hf_subset":"gu","hf_avail_splits":["test","dev"],"evaluation_splits":["test"],"few_shots_split":"dev","few_shots_select":"random_sampling_from_train","generation_size":1,"metric":["loglikelihood_acc","loglikelihood_acc_norm_nospace"],"stop_sequence":["\n"],"output_regex":null,"frozen":false, "trust_dataset": true} -{"name":"Flores:marathi","suite":["indiceval","leaderboard","flores"],"prompt_function":"flores_indic","hf_repo":"Cognitive-Lab\/GoogleIndicGenBench_flores_in","hf_subset":"mr","hf_avail_splits":["test","dev"],"evaluation_splits":["test"],"few_shots_split":"dev","few_shots_select":"random_sampling_from_train","generation_size":1,"metric":["loglikelihood_acc","loglikelihood_acc_norm_nospace"],"stop_sequence":["\n"],"output_regex":null,"frozen":false, "trust_dataset": true} -{"name":"Flores:malayalam","suite":["indiceval","leaderboard","flores"],"prompt_function":"flores_indic","hf_repo":"Cognitive-Lab\/GoogleIndicGenBench_flores_in","hf_subset":"ml","hf_avail_splits":["test","dev"],"evaluation_splits":["test"],"few_shots_split":"dev","few_shots_select":"random_sampling_from_train","generation_size":1,"metric":["loglikelihood_acc","loglikelihood_acc_norm_nospace"],"stop_sequence":["\n"],"output_regex":null,"frozen":false, "trust_dataset": true} +{"name":"Flores_enxx:hindi","suite":["indiceval","leaderboard","flores_enxx"],"prompt_function":"flores_enxx_indic","hf_repo":"Cognitive-Lab\/GoogleIndicGenBench_flores_enxx_in","hf_subset":"hi","hf_avail_splits":["test","dev"],"evaluation_splits":["test"],"few_shots_split":"dev","few_shots_select":"random_sampling_from_train","generation_size":1,"metric":["loglikelihood_acc","loglikelihood_acc_norm_nospace"],"stop_sequence":["\n"],"output_regex":null,"frozen":false, "trust_dataset": true} +{"name":"Flores_enxx:kannada","suite":["indiceval","leaderboard","flores_enxx"],"prompt_function":"flores_enxx_indic","hf_repo":"Cognitive-Lab\/GoogleIndicGenBench_flores_enxx_in","hf_subset":"kn","hf_avail_splits":["test","dev"],"evaluation_splits":["test"],"few_shots_split":"dev","few_shots_select":"random_sampling_from_train","generation_size":1,"metric":["loglikelihood_acc","loglikelihood_acc_norm_nospace"],"stop_sequence":["\n"],"output_regex":null,"frozen":false, "trust_dataset": true} +{"name":"Flores_enxx:tamil","suite":["indiceval","leaderboard","flores_enxx"],"prompt_function":"flores_enxx_indic","hf_repo":"Cognitive-Lab\/GoogleIndicGenBench_flores_enxx_in","hf_subset":"ta","hf_avail_splits":["test","dev"],"evaluation_splits":["test"],"few_shots_split":"dev","few_shots_select":"random_sampling_from_train","generation_size":1,"metric":["loglikelihood_acc","loglikelihood_acc_norm_nospace"],"stop_sequence":["\n"],"output_regex":null,"frozen":false, "trust_dataset": true} +{"name":"Flores_enxx:telugu","suite":["indiceval","leaderboard","flores_enxx"],"prompt_function":"flores_enxx_indic","hf_repo":"Cognitive-Lab\/GoogleIndicGenBench_flores_enxx_in","hf_subset":"te","hf_avail_splits":["test","dev"],"evaluation_splits":["test"],"few_shots_split":"dev","few_shots_select":"random_sampling_from_train","generation_size":1,"metric":["loglikelihood_acc","loglikelihood_acc_norm_nospace"],"stop_sequence":["\n"],"output_regex":null,"frozen":false, "trust_dataset": true} +{"name":"Flores_enxx:gujarati","suite":["indiceval","leaderboard","flores_enxx"],"prompt_function":"flores_enxx_indic","hf_repo":"Cognitive-Lab\/GoogleIndicGenBench_flores_enxx_in","hf_subset":"gu","hf_avail_splits":["test","dev"],"evaluation_splits":["test"],"few_shots_split":"dev","few_shots_select":"random_sampling_from_train","generation_size":1,"metric":["loglikelihood_acc","loglikelihood_acc_norm_nospace"],"stop_sequence":["\n"],"output_regex":null,"frozen":false, "trust_dataset": true} +{"name":"Flores_enxx:marathi","suite":["indiceval","leaderboard","flores_enxx"],"prompt_function":"flores_enxx_indic","hf_repo":"Cognitive-Lab\/GoogleIndicGenBench_flores_enxx_in","hf_subset":"mr","hf_avail_splits":["test","dev"],"evaluation_splits":["test"],"few_shots_split":"dev","few_shots_select":"random_sampling_from_train","generation_size":1,"metric":["loglikelihood_acc","loglikelihood_acc_norm_nospace"],"stop_sequence":["\n"],"output_regex":null,"frozen":false, "trust_dataset": true} +{"name":"Flores_enxx:malayalam","suite":["indiceval","leaderboard","flores_enxx"],"prompt_function":"flores_enxx_indic","hf_repo":"Cognitive-Lab\/GoogleIndicGenBench_flores_enxx_in","hf_subset":"ml","hf_avail_splits":["test","dev"],"evaluation_splits":["test"],"few_shots_split":"dev","few_shots_select":"random_sampling_from_train","generation_size":1,"metric":["loglikelihood_acc","loglikelihood_acc_norm_nospace"],"stop_sequence":["\n"],"output_regex":null,"frozen":false, "trust_dataset": true} + +{"name":"Flores_xxen:hindi","suite":["indiceval","leaderboard","flores_xxen"],"prompt_function":"flores_xxen_indic","hf_repo":"Cognitive-Lab\/GoogleIndicGenBench_flores_xxen_in","hf_subset":"hi","hf_avail_splits":["test","dev"],"evaluation_splits":["test"],"few_shots_split":"dev","few_shots_select":"random_sampling_from_train","generation_size":1,"metric":["loglikelihood_acc","loglikelihood_acc_norm_nospace"],"stop_sequence":["\n"],"output_regex":null,"frozen":false, "trust_dataset": true} +{"name":"Flores_xxen:kannada","suite":["indiceval","leaderboard","flores_xxen"],"prompt_function":"flores_xxen_indic","hf_repo":"Cognitive-Lab\/GoogleIndicGenBench_flores_xxen_in","hf_subset":"kn","hf_avail_splits":["test","dev"],"evaluation_splits":["test"],"few_shots_split":"dev","few_shots_select":"random_sampling_from_train","generation_size":1,"metric":["loglikelihood_acc","loglikelihood_acc_norm_nospace"],"stop_sequence":["\n"],"output_regex":null,"frozen":false, "trust_dataset": true} +{"name":"Flores_xxen:tamil","suite":["indiceval","leaderboard","flores_xxen"],"prompt_function":"flores_xxen_indic","hf_repo":"Cognitive-Lab\/GoogleIndicGenBench_flores_xxen_in","hf_subset":"ta","hf_avail_splits":["test","dev"],"evaluation_splits":["test"],"few_shots_split":"dev","few_shots_select":"random_sampling_from_train","generation_size":1,"metric":["loglikelihood_acc","loglikelihood_acc_norm_nospace"],"stop_sequence":["\n"],"output_regex":null,"frozen":false, "trust_dataset": true} +{"name":"Flores_xxen:telugu","suite":["indiceval","leaderboard","flores_xxen"],"prompt_function":"flores_xxen_indic","hf_repo":"Cognitive-Lab\/GoogleIndicGenBench_flores_xxen_in","hf_subset":"te","hf_avail_splits":["test","dev"],"evaluation_splits":["test"],"few_shots_split":"dev","few_shots_select":"random_sampling_from_train","generation_size":1,"metric":["loglikelihood_acc","loglikelihood_acc_norm_nospace"],"stop_sequence":["\n"],"output_regex":null,"frozen":false, "trust_dataset": true} +{"name":"Flores_xxen:gujarati","suite":["indiceval","leaderboard","flores_xxen"],"prompt_function":"flores_xxen_indic","hf_repo":"Cognitive-Lab\/GoogleIndicGenBench_flores_xxen_in","hf_subset":"gu","hf_avail_splits":["test","dev"],"evaluation_splits":["test"],"few_shots_split":"dev","few_shots_select":"random_sampling_from_train","generation_size":1,"metric":["loglikelihood_acc","loglikelihood_acc_norm_nospace"],"stop_sequence":["\n"],"output_regex":null,"frozen":false, "trust_dataset": true} +{"name":"Flores_xxen:marathi","suite":["indiceval","leaderboard","flores_xxen"],"prompt_function":"flores_xxen_indic","hf_repo":"Cognitive-Lab\/GoogleIndicGenBench_flores_xxen_in","hf_subset":"mr","hf_avail_splits":["test","dev"],"evaluation_splits":["test"],"few_shots_split":"dev","few_shots_select":"random_sampling_from_train","generation_size":1,"metric":["loglikelihood_acc","loglikelihood_acc_norm_nospace"],"stop_sequence":["\n"],"output_regex":null,"frozen":false, "trust_dataset": true} +{"name":"Flores_xxen:malayalam","suite":["indiceval","leaderboard","flores_xxen"],"prompt_function":"flores_xxen_indic","hf_repo":"Cognitive-Lab\/GoogleIndicGenBench_flores_xxen_in","hf_subset":"ml","hf_avail_splits":["test","dev"],"evaluation_splits":["test"],"few_shots_split":"dev","few_shots_select":"random_sampling_from_train","generation_size":1,"metric":["loglikelihood_acc","loglikelihood_acc_norm_nospace"],"stop_sequence":["\n"],"output_regex":null,"frozen":false, "trust_dataset": true} {"name":"Crosssum:hindi","suite":["indiceval","leaderboard","crosssum"],"prompt_function":"crosssum_indic","hf_repo":"Cognitive-Lab\/GoogleIndicGenBench_crosssum_in","hf_subset":"hi","hf_avail_splits":["test","dev"],"evaluation_splits":["test"],"few_shots_split":"dev","few_shots_select":"random_sampling_from_train","generation_size":1,"metric":["loglikelihood_acc","loglikelihood_acc_norm_nospace"],"stop_sequence":["\n"],"output_regex":null,"frozen":false, "trust_dataset": true} {"name":"Crosssum:kannada","suite":["indiceval","leaderboard","crosssum"],"prompt_function":"crosssum_indic","hf_repo":"Cognitive-Lab\/GoogleIndicGenBench_crosssum_in","hf_subset":"kn","hf_avail_splits":["test","dev"],"evaluation_splits":["test"],"few_shots_split":"dev","few_shots_select":"random_sampling_from_train","generation_size":1,"metric":["loglikelihood_acc","loglikelihood_acc_norm_nospace"],"stop_sequence":["\n"],"output_regex":null,"frozen":false, "trust_dataset": true}