data_filtering.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "022fb7df-e22f-4512-be01-50f8788d2160",
   "metadata": {},
   "outputs": [],
   "source": [
    "from dataclasses import dataclass\n",
    "import pandas as pd\n",
    "from nusacrowd import NusantaraMetadata, NusantaraConfigHelper, NusantaraMetadataHelper\n",
    "from nusacrowd.utils.constants import Tasks, TASK_TO_SCHEMA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4b5ceeab-64f3-4afb-8539-de28ae08dc20",
   "metadata": {
    "scrolled": true,
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Nusantara Speech public datasets\n",
      "NusantaraMetadata(script='/home/samuel/anaconda2/envs/env_nusa_exp/lib/python3.8/site-packages/nusacrowd/nusa_datasets/covost2/covost2.py', dataset_name='covost2', tasks={<Tasks.SPEECH_TO_TEXT_TRANSLATION: 'STTT'>}, languages=['ind', 'eng'], config=NusantaraConfig(name='covost2_ind_eng_nusantara_sptext', version=1.0.0, data_dir=None, data_files=None, description='covost2 source schema for nusantara_sptext from ind to eng', schema='nusantara_sptext', subset_id='co_vo_st2_ind_eng'), is_local=False, is_nusantara_schema=True, nusantara_schema_caps='SPTEXT', is_large=False, is_resource=False, is_default=False, is_broken=False, nusantara_version='1.0.0', source_version='1.0.0', citation='\\n@article{wang2020covost,\\n  title={Covost 2 and massively multilingual speech-to-text translation},\\n  author={Wang, Changhan and Wu, Anne and Pino, Juan},\\n  journal={arXiv preprint arXiv:2007.10310},\\n  year={2020}\\n}\\n\\n@inproceedings{wang21s_interspeech,\\n  author={Wang, Changhan and Wu, Anne and Pino, Juan},\\n  title={{CoVoST 2 and Massively Multilingual Speech Translation}},\\n  year=2021,\\n  booktitle={Proc. Interspeech 2021},\\n  pages={2247--2251},\\n  url={https://www.isca-speech.org/archive/interspeech_2021/wang21s_interspeech}\\n  doi={10.21437/Interspeech.2021-2027}\\n}\\n', description=\"CoVoST2 is a large-scale multilingual speech translation corpus covering translations from 21 languages to English\\nand from English into 15 languages. The dataset is created using Mozilla's open-source Common Voice database of\\ncrowdsourced voice recordings. There are 2,900 hours of speech represented in the corpus.\\n\", homepage='https://huggingface.co/datasets/covost2', license='CC BY-NC 4.0')\n",
      "\n",
      "NusantaraMetadata(script='/home/samuel/anaconda2/envs/env_nusa_exp/lib/python3.8/site-packages/nusacrowd/nusa_datasets/covost2/covost2.py', dataset_name='covost2', tasks={<Tasks.SPEECH_TO_TEXT_TRANSLATION: 'STTT'>}, languages=['ind', 'eng'], config=NusantaraConfig(name='covost2_eng_ind_nusantara_sptext', version=1.0.0, data_dir=None, data_files=None, description='covost2 source schema for nusantara_sptext from eng to ind', schema='nusantara_sptext', subset_id='co_vo_st2_eng_ind'), is_local=False, is_nusantara_schema=True, nusantara_schema_caps='SPTEXT', is_large=False, is_resource=False, is_default=False, is_broken=False, nusantara_version='1.0.0', source_version='1.0.0', citation='\\n@article{wang2020covost,\\n  title={Covost 2 and massively multilingual speech-to-text translation},\\n  author={Wang, Changhan and Wu, Anne and Pino, Juan},\\n  journal={arXiv preprint arXiv:2007.10310},\\n  year={2020}\\n}\\n\\n@inproceedings{wang21s_interspeech,\\n  author={Wang, Changhan and Wu, Anne and Pino, Juan},\\n  title={{CoVoST 2 and Massively Multilingual Speech Translation}},\\n  year=2021,\\n  booktitle={Proc. Interspeech 2021},\\n  pages={2247--2251},\\n  url={https://www.isca-speech.org/archive/interspeech_2021/wang21s_interspeech}\\n  doi={10.21437/Interspeech.2021-2027}\\n}\\n', description=\"CoVoST2 is a large-scale multilingual speech translation corpus covering translations from 21 languages to English\\nand from English into 15 languages. The dataset is created using Mozilla's open-source Common Voice database of\\ncrowdsourced voice recordings. There are 2,900 hours of speech represented in the corpus.\\n\", homepage='https://huggingface.co/datasets/covost2', license='CC BY-NC 4.0')\n",
      "\n",
      "NusantaraMetadata(script='/home/samuel/anaconda2/envs/env_nusa_exp/lib/python3.8/site-packages/nusacrowd/nusa_datasets/cvss/cvss.py', dataset_name='cvss', tasks={<Tasks.SPEECH_TO_SPEECH_TRANSLATION: 'S2ST'>}, languages=['ind', 'eng'], config=NusantaraConfig(name='cvss_c_nusantara_s2s', version=1.0.0, data_dir=None, data_files=None, description=\"CVSS Nusantara schema, all translation speeches are in a single canonical speaker's voice.\", schema='nusantara_s2s', subset_id='cvss_c'), is_local=False, is_nusantara_schema=True, nusantara_schema_caps='S2S', is_large=False, is_resource=False, is_default=False, is_broken=False, nusantara_version='1.0.0', source_version='1.0.0', citation='@inproceedings{jia2022cvss,\\n    title={{CVSS} Corpus and Massively Multilingual Speech-to-Speech Translation},\\n    author={Jia, Ye and Tadmor Ramanovich, Michelle and Wang, Quan and Zen, Heiga},\\n    booktitle={Proceedings of Language Resources and Evaluation Conference (LREC)},\\n    pages={6691--6703},\\n    year={2022}\\n}\\n', description='CVSS is a massively multilingual-to-English speech-to-speech translation corpus,\\ncovering sentence-level parallel speech-to-speech translation pairs from 21\\nlanguages into English.\\n', homepage='https://github.com/google-research-datasets/cvss', license='CC-BY 4.0')\n",
      "\n",
      "NusantaraMetadata(script='/home/samuel/anaconda2/envs/env_nusa_exp/lib/python3.8/site-packages/nusacrowd/nusa_datasets/cvss/cvss.py', dataset_name='cvss', tasks={<Tasks.SPEECH_TO_SPEECH_TRANSLATION: 'S2ST'>}, languages=['ind', 'eng'], config=NusantaraConfig(name='cvss_t_nusantara_s2s', version=1.0.0, data_dir=None, data_files=None, description='CVSS Nusantara schema, translation speeches are in voices transferred from the corresponding source speeches', schema='nusantara_s2s', subset_id='cvss_t'), is_local=False, is_nusantara_schema=True, nusantara_schema_caps='S2S', is_large=False, is_resource=False, is_default=False, is_broken=False, nusantara_version='1.0.0', source_version='1.0.0', citation='@inproceedings{jia2022cvss,\\n    title={{CVSS} Corpus and Massively Multilingual Speech-to-Speech Translation},\\n    author={Jia, Ye and Tadmor Ramanovich, Michelle and Wang, Quan and Zen, Heiga},\\n    booktitle={Proceedings of Language Resources and Evaluation Conference (LREC)},\\n    pages={6691--6703},\\n    year={2022}\\n}\\n', description='CVSS is a massively multilingual-to-English speech-to-speech translation corpus,\\ncovering sentence-level parallel speech-to-speech translation pairs from 21\\nlanguages into English.\\n', homepage='https://github.com/google-research-datasets/cvss', license='CC-BY 4.0')\n",
      "\n",
      "NusantaraMetadata(script='/home/samuel/anaconda2/envs/env_nusa_exp/lib/python3.8/site-packages/nusacrowd/nusa_datasets/indspeech_digit_cdsr/indspeech_digit_cdsr.py', dataset_name='indspeech_digit_cdsr', tasks={<Tasks.SPEECH_RECOGNITION: 'ASR'>}, languages=['ind'], config=NusantaraConfig(name='indspeech_digit_cdsr_nusantara_sptext', version=1.0.0, data_dir=None, data_files=None, description='indspeech_digit_cdsr Nusantara schema', schema='nusantara_sptext', subset_id='indspeech_digit_cdsr'), is_local=False, is_nusantara_schema=True, nusantara_schema_caps='SPTEXT', is_large=False, is_resource=False, is_default=False, is_broken=False, nusantara_version='1.0.0', source_version='1.0.0', citation='@inproceedings{sakti-icslp-2004,\\n    title = \"Indonesian Speech Recognition for Hearing and Speaking Impaired People\",\\n    author = \"Sakti, Sakriani and Hutagaol, Paulus and Arman, Arry Akhmad and Nakamura, Satoshi\",\\n    booktitle = \"Proc. International Conference on Spoken Language Processing (INTERSPEECH - ICSLP)\",\\n    year = \"2004\",\\n    pages = \"1037--1040\"\\n    address = \"Jeju Island, Korea\"\\n}\\n', description='INDspeech_DIGIT_CDSR is the first Indonesian speech dataset for connected digit speech recognition (CDSR). The data was developed by TELKOMRisTI (R&D Division, PT Telekomunikasi Indonesia) in collaboration with Advanced Telecommunication Research Institute International (ATR) Japan and Bandung Institute of Technology (ITB) under the Asia-Pacific Telecommunity (APT) project in 2004 [Sakti et al., 2004]. Although it was originally developed for a telecommunication system for hearing and speaking impaired people, it can be used for other applications, i.e., automatic call centers that recognize telephone numbers.\\n', homepage='https://github.com/s-sakti/data_indsp_digit_cdsr', license='CC-BY-NC-SA-4.0')\n",
      "\n",
      "NusantaraMetadata(script='/home/samuel/anaconda2/envs/env_nusa_exp/lib/python3.8/site-packages/nusacrowd/nusa_datasets/indspeech_news_ethnicsr/indspeech_news_ethnicsr.py', dataset_name='indspeech_news_ethnicsr', tasks={<Tasks.SPEECH_RECOGNITION: 'ASR'>}, languages=['ind'], config=NusantaraConfig(name='indspeech_news_ethnicsr_jv_overlap_nusantara_sptext', version='1.0.0', data_dir=None, data_files=None, description='indspeech_news_ethnicsr Nusantara schema', schema='nusantara_sptext', subset_id='indspeech_news_ethnicsr_jv_overlap'), is_local=False, is_nusantara_schema=True, nusantara_schema_caps='SPTEXT', is_large=False, is_resource=False, is_default=False, is_broken=False, nusantara_version='1.0.0', source_version='1.0.0', citation='@inproceedings{sani-cocosda-2012,\\n    title = \"Towards Language Preservation: Preliminary Collection and Vowel Analysis of {I}ndonesian Ethnic Speech Data\",\\n    author = \"Sani, Auliya and Sakti, Sakriani and Neubig, Graham and Toda, Tomoki and Mulyanto, Adi and Nakamura, Satoshi\",\\n    booktitle = \"Proc. Oriental COCOSDA\",\\n    year = \"2012\",\\n    pages = \"118--122\"\\n    address = \"Macau, China\"\\n}\\n', description='\\nINDspeech_NEWS_EthnicSR is a collection of Indonesian ethnic speech corpora for Javanese and Sundanese for Indonesian ethnic speech recognition. It was developed in 2012 by the Nara Institute of Science and Technology (NAIST, Japan) in collaboration with the Bandung Institute of Technology (ITB, Indonesia) [Sani et al., 2012].\\n', homepage='https://github.com/s-sakti/data_indsp_news_ethnicsr', license='CC-BY-NC-SA 4.0')\n",
      "\n",
      "NusantaraMetadata(script='/home/samuel/anaconda2/envs/env_nusa_exp/lib/python3.8/site-packages/nusacrowd/nusa_datasets/indspeech_news_ethnicsr/indspeech_news_ethnicsr.py', dataset_name='indspeech_news_ethnicsr', tasks={<Tasks.SPEECH_RECOGNITION: 'ASR'>}, languages=['ind'], config=NusantaraConfig(name='indspeech_news_ethnicsr_su_overlap_nusantara_sptext', version='1.0.0', data_dir=None, data_files=None, description='indspeech_news_ethnicsr Nusantara schema', schema='nusantara_sptext', subset_id='indspeech_news_ethnicsr_su_overlap'), is_local=False, is_nusantara_schema=True, nusantara_schema_caps='SPTEXT', is_large=False, is_resource=False, is_default=False, is_broken=False, nusantara_version='1.0.0', source_version='1.0.0', citation='@inproceedings{sani-cocosda-2012,\\n    title = \"Towards Language Preservation: Preliminary Collection and Vowel Analysis of {I}ndonesian Ethnic Speech Data\",\\n    author = \"Sani, Auliya and Sakti, Sakriani and Neubig, Graham and Toda, Tomoki and Mulyanto, Adi and Nakamura, Satoshi\",\\n    booktitle = \"Proc. Oriental COCOSDA\",\\n    year = \"2012\",\\n    pages = \"118--122\"\\n    address = \"Macau, China\"\\n}\\n', description='\\nINDspeech_NEWS_EthnicSR is a collection of Indonesian ethnic speech corpora for Javanese and Sundanese for Indonesian ethnic speech recognition. It was developed in 2012 by the Nara Institute of Science and Technology (NAIST, Japan) in collaboration with the Bandung Institute of Technology (ITB, Indonesia) [Sani et al., 2012].\\n', homepage='https://github.com/s-sakti/data_indsp_news_ethnicsr', license='CC-BY-NC-SA 4.0')\n",
      "\n",
      "NusantaraMetadata(script='/home/samuel/anaconda2/envs/env_nusa_exp/lib/python3.8/site-packages/nusacrowd/nusa_datasets/indspeech_news_ethnicsr/indspeech_news_ethnicsr.py', dataset_name='indspeech_news_ethnicsr', tasks={<Tasks.SPEECH_RECOGNITION: 'ASR'>}, languages=['ind'], config=NusantaraConfig(name='indspeech_news_ethnicsr_jv_nooverlap_nusantara_sptext', version='1.0.0', data_dir=None, data_files=None, description='indspeech_news_ethnicsr Nusantara schema', schema='nusantara_sptext', subset_id='indspeech_news_ethnicsr_jv_nooverlap'), is_local=False, is_nusantara_schema=True, nusantara_schema_caps='SPTEXT', is_large=False, is_resource=False, is_default=False, is_broken=False, nusantara_version='1.0.0', source_version='1.0.0', citation='@inproceedings{sani-cocosda-2012,\\n    title = \"Towards Language Preservation: Preliminary Collection and Vowel Analysis of {I}ndonesian Ethnic Speech Data\",\\n    author = \"Sani, Auliya and Sakti, Sakriani and Neubig, Graham and Toda, Tomoki and Mulyanto, Adi and Nakamura, Satoshi\",\\n    booktitle = \"Proc. Oriental COCOSDA\",\\n    year = \"2012\",\\n    pages = \"118--122\"\\n    address = \"Macau, China\"\\n}\\n', description='\\nINDspeech_NEWS_EthnicSR is a collection of Indonesian ethnic speech corpora for Javanese and Sundanese for Indonesian ethnic speech recognition. It was developed in 2012 by the Nara Institute of Science and Technology (NAIST, Japan) in collaboration with the Bandung Institute of Technology (ITB, Indonesia) [Sani et al., 2012].\\n', homepage='https://github.com/s-sakti/data_indsp_news_ethnicsr', license='CC-BY-NC-SA 4.0')\n",
      "\n",
      "NusantaraMetadata(script='/home/samuel/anaconda2/envs/env_nusa_exp/lib/python3.8/site-packages/nusacrowd/nusa_datasets/indspeech_news_ethnicsr/indspeech_news_ethnicsr.py', dataset_name='indspeech_news_ethnicsr', tasks={<Tasks.SPEECH_RECOGNITION: 'ASR'>}, languages=['ind'], config=NusantaraConfig(name='indspeech_news_ethnicsr_su_nooverlap_nusantara_sptext', version='1.0.0', data_dir=None, data_files=None, description='indspeech_news_ethnicsr Nusantara schema', schema='nusantara_sptext', subset_id='indspeech_news_ethnicsr_su_nooverlap'), is_local=False, is_nusantara_schema=True, nusantara_schema_caps='SPTEXT', is_large=False, is_resource=False, is_default=False, is_broken=False, nusantara_version='1.0.0', source_version='1.0.0', citation='@inproceedings{sani-cocosda-2012,\\n    title = \"Towards Language Preservation: Preliminary Collection and Vowel Analysis of {I}ndonesian Ethnic Speech Data\",\\n    author = \"Sani, Auliya and Sakti, Sakriani and Neubig, Graham and Toda, Tomoki and Mulyanto, Adi and Nakamura, Satoshi\",\\n    booktitle = \"Proc. Oriental COCOSDA\",\\n    year = \"2012\",\\n    pages = \"118--122\"\\n    address = \"Macau, China\"\\n}\\n', description='\\nINDspeech_NEWS_EthnicSR is a collection of Indonesian ethnic speech corpora for Javanese and Sundanese for Indonesian ethnic speech recognition. It was developed in 2012 by the Nara Institute of Science and Technology (NAIST, Japan) in collaboration with the Bandung Institute of Technology (ITB, Indonesia) [Sani et al., 2012].\\n', homepage='https://github.com/s-sakti/data_indsp_news_ethnicsr', license='CC-BY-NC-SA 4.0')\n",
      "\n",
      "NusantaraMetadata(script='/home/samuel/anaconda2/envs/env_nusa_exp/lib/python3.8/site-packages/nusacrowd/nusa_datasets/indspeech_news_lvcsr/indspeech_news_lvcsr.py', dataset_name='indspeech_news_lvcsr', tasks={<Tasks.SPEECH_RECOGNITION: 'ASR'>}, languages=['ind'], config=NusantaraConfig(name='indspeech_news_lvcsr_nusantara_sptext', version=1.0.0, data_dir=None, data_files=None, description='indspeech_news_lvcsr Nusantara schema', schema='nusantara_sptext', subset_id='indspeech_news_lvcsr'), is_local=False, is_nusantara_schema=True, nusantara_schema_caps='SPTEXT', is_large=False, is_resource=False, is_default=False, is_broken=False, nusantara_version='1.0.0', source_version='1.0.0', citation='@inproceedings{sakti-tcast-2008,\\n    title = \"Development of {I}ndonesian Large Vocabulary Continuous Speech Recognition System within {A-STAR} Project\",\\n    author = \"Sakti, Sakriani and Kelana, Eka and Riza, Hammam and Sakai, Shinsuke and Markov, Konstantin and Nakamura, Satoshi\",\\n    booktitle = \"Proc. IJCNLP Workshop on Technologies and Corpora for Asia-Pacific Speech Translation (TCAST)\",\\n    year = \"2008\",\\n    pages = \"19--24\"\\n    address = \"Hyderabad, India\"\\n}\\n\\n@inproceedings{sakti-icslp-2004,\\n    title = \"Indonesian Speech Recognition for Hearing and Speaking Impaired People\",\\n    author = \"Sakti, Sakriani and Hutagaol, Paulus and Arman, Arry Akhmad and Nakamura, Satoshi\",\\n    booktitle = \"Proc. International Conference on Spoken Language Processing (INTERSPEECH - ICSLP)\",\\n    year = \"2004\",\\n    pages = \"1037--1040\"\\n    address = \"Jeju Island, Korea\"\\n}\\n\\n@article{sakti-s2st-csl-2013,\\n    title = \"{A-STAR}: Toward Translating Asian Spoken Languages\",\\n    author = \"Sakti, Sakriani and Paul, Michael and Finch, Andrew and Sakai, Shinsuke and Thang, Tat Vu, and Kimura, Noriyuki and Hori, Chiori and Sumita, Eiichiro and Nakamura, Satoshi and Park, Jun and Wutiwiwatchai, Chai and Xu, Bo and Riza, Hammam and Arora, Karunesh and Luong, Chi Mai and Li, Haizhou\",\\n    journal = \"Special issue on Speech-to-Speech Translation, Computer Speech and Language Journal\",\\n    volume = \"27\",\\n    number =\"2\",\\n    pages = \"509--527\",\\n    year = \"2013\",\\n    publisher = \"Elsevier\"\\n}\\n', description='This is the first Indonesian speech dataset for large vocabulary continuous speech recognition (LVCSR) with more than 40 hours of speech and 400 speakers [Sakti et al., 2008]. R&D Division of PT Telekomunikasi Indonesia (TELKOMRisTI) developed the data in 2005-2006, in collaboration with Advanced Telecommunication Research Institute International (ATR) Japan, as the continuation of the Asia-Pacific Telecommunity (APT) project [Sakti et al., 2004]. It has also been successfully used for developing Indonesian LVCSR in the Asian speech translation advanced research (A-STAR) project [Sakti et al., 2013].\\n', homepage='https://github.com/s-sakti/data_indsp_news_lvcsr', license='CC BY-NC-SA 4.0')\n",
      "\n",
      "NusantaraMetadata(script='/home/samuel/anaconda2/envs/env_nusa_exp/lib/python3.8/site-packages/nusacrowd/nusa_datasets/indspeech_news_tts/indspeech_news_tts.py', dataset_name='indspeech_news_tts', tasks={<Tasks.SPEECH_RECOGNITION: 'ASR'>}, languages=['ind'], config=NusantaraConfig(name='indspeech_news_tts_12_MOS_nusantara_sptext', version=1.0.0, data_dir=None, data_files=None, description='indspeech_news_tts nusantara_sptext schema for 12 train and MOS test task', schema='nusantara_sptext', subset_id='indspeech_news_tts_12_MOS'), is_local=False, is_nusantara_schema=True, nusantara_schema_caps='SPTEXT', is_large=False, is_resource=False, is_default=False, is_broken=False, nusantara_version='1.0.0', source_version='1.0.0', citation='@inproceedings{sakti-tts-cocosda-2008,\\n    title = \"Development of HMM-based Indonesian Speech Synthesis\",\\n    author = \"Sakti, Sakriani and Maia, Ranniery and Sakai, Shinsuke and Nakamura, Satoshi\",\\n    booktitle = \"Proc. Oriental COCOSDA\",\\n    year = \"2008\",\\n    pages = \"215--220\"\\n    address = \"Kyoto, Japan\"\\n}\\n\\n@inproceedings{sakti-tts-malindo-2010,\\n    title = \"Quality and Intelligibility Assessment of Indonesian HMM-Based Speech Synthesis System\",\\n    author = \"Sakti, Sakriani and Sakai, Shinsuke and Isotani, Ryosuke and Kawai, Hisashi and Nakamura, Satoshi\",\\n    booktitle = \"Proc. MALINDO\",\\n    year = \"2010\",\\n    pages = \"51--57\"\\n    address = \"Jakarta, Indonesia\"\\n}\\n\\n@article{sakti-s2st-csl-2013,\\n    title = \"{A-STAR}: Toward Tranlating Asian Spoken Languages\",\\n    author = \"Sakti, Sakriani and Paul, Michael and Finch, Andrew and Sakai, Shinsuke and Thang, Tat Vu, and Kimura, Noriyuki \\n    and Hori, Chiori and Sumita, Eiichiro and Nakamura, Satoshi and Park, Jun and Wutiwiwatchai, Chai and Xu, Bo and Riza, Hammam \\n    and Arora, Karunesh and Luong, Chi Mai and Li, Haizhou\",\\n    journal = \"Special issue on Speech-to-Speech Translation, Computer Speech and Language Journal\",\\n    volume = \"27\",\\n    number =\"2\",\\n    pages = \"509--527\",\\n    year = \"2013\",\\n    publisher = \"Elsevier\"\\n}\\n', description='INDspeech_NEWS_TTS is a speech dataset for developing an Indonesian text-to-speech synthesis system. The data was developed by Advanced Telecommunication Research Institute International (ATR) Japan under the the Asian speech translation advanced research (A-STAR) project [Sakti et al., 2013].\\n', homepage='https://github.com/s-sakti/data_indsp_news_tts', license='CC-BY-NC-SA 4.0')\n",
      "\n",
      "NusantaraMetadata(script='/home/samuel/anaconda2/envs/env_nusa_exp/lib/python3.8/site-packages/nusacrowd/nusa_datasets/indspeech_news_tts/indspeech_news_tts.py', dataset_name='indspeech_news_tts', tasks={<Tasks.SPEECH_RECOGNITION: 'ASR'>}, languages=['ind'], config=NusantaraConfig(name='indspeech_news_tts_30_MOS_nusantara_sptext', version=1.0.0, data_dir=None, data_files=None, description='indspeech_news_tts nusantara_sptext schema for 30 train and MOS test task', schema='nusantara_sptext', subset_id='indspeech_news_tts_30_MOS'), is_local=False, is_nusantara_schema=True, nusantara_schema_caps='SPTEXT', is_large=False, is_resource=False, is_default=False, is_broken=False, nusantara_version='1.0.0', source_version='1.0.0', citation='@inproceedings{sakti-tts-cocosda-2008,\\n    title = \"Development of HMM-based Indonesian Speech Synthesis\",\\n    author = \"Sakti, Sakriani and Maia, Ranniery and Sakai, Shinsuke and Nakamura, Satoshi\",\\n    booktitle = \"Proc. Oriental COCOSDA\",\\n    year = \"2008\",\\n    pages = \"215--220\"\\n    address = \"Kyoto, Japan\"\\n}\\n\\n@inproceedings{sakti-tts-malindo-2010,\\n    title = \"Quality and Intelligibility Assessment of Indonesian HMM-Based Speech Synthesis System\",\\n    author = \"Sakti, Sakriani and Sakai, Shinsuke and Isotani, Ryosuke and Kawai, Hisashi and Nakamura, Satoshi\",\\n    booktitle = \"Proc. MALINDO\",\\n    year = \"2010\",\\n    pages = \"51--57\"\\n    address = \"Jakarta, Indonesia\"\\n}\\n\\n@article{sakti-s2st-csl-2013,\\n    title = \"{A-STAR}: Toward Tranlating Asian Spoken Languages\",\\n    author = \"Sakti, Sakriani and Paul, Michael and Finch, Andrew and Sakai, Shinsuke and Thang, Tat Vu, and Kimura, Noriyuki \\n    and Hori, Chiori and Sumita, Eiichiro and Nakamura, Satoshi and Park, Jun and Wutiwiwatchai, Chai and Xu, Bo and Riza, Hammam \\n    and Arora, Karunesh and Luong, Chi Mai and Li, Haizhou\",\\n    journal = \"Special issue on Speech-to-Speech Translation, Computer Speech and Language Journal\",\\n    volume = \"27\",\\n    number =\"2\",\\n    pages = \"509--527\",\\n    year = \"2013\",\\n    publisher = \"Elsevier\"\\n}\\n', description='INDspeech_NEWS_TTS is a speech dataset for developing an Indonesian text-to-speech synthesis system. The data was developed by Advanced Telecommunication Research Institute International (ATR) Japan under the the Asian speech translation advanced research (A-STAR) project [Sakti et al., 2013].\\n', homepage='https://github.com/s-sakti/data_indsp_news_tts', license='CC-BY-NC-SA 4.0')\n",
      "\n",
      "NusantaraMetadata(script='/home/samuel/anaconda2/envs/env_nusa_exp/lib/python3.8/site-packages/nusacrowd/nusa_datasets/indspeech_news_tts/indspeech_news_tts.py', dataset_name='indspeech_news_tts', tasks={<Tasks.SPEECH_RECOGNITION: 'ASR'>}, languages=['ind'], config=NusantaraConfig(name='indspeech_news_tts_60_MOS_nusantara_sptext', version=1.0.0, data_dir=None, data_files=None, description='indspeech_news_tts nusantara_sptext schema for 60 train and MOS test task', schema='nusantara_sptext', subset_id='indspeech_news_tts_60_MOS'), is_local=False, is_nusantara_schema=True, nusantara_schema_caps='SPTEXT', is_large=False, is_resource=False, is_default=False, is_broken=False, nusantara_version='1.0.0', source_version='1.0.0', citation='@inproceedings{sakti-tts-cocosda-2008,\\n    title = \"Development of HMM-based Indonesian Speech Synthesis\",\\n    author = \"Sakti, Sakriani and Maia, Ranniery and Sakai, Shinsuke and Nakamura, Satoshi\",\\n    booktitle = \"Proc. Oriental COCOSDA\",\\n    year = \"2008\",\\n    pages = \"215--220\"\\n    address = \"Kyoto, Japan\"\\n}\\n\\n@inproceedings{sakti-tts-malindo-2010,\\n    title = \"Quality and Intelligibility Assessment of Indonesian HMM-Based Speech Synthesis System\",\\n    author = \"Sakti, Sakriani and Sakai, Shinsuke and Isotani, Ryosuke and Kawai, Hisashi and Nakamura, Satoshi\",\\n    booktitle = \"Proc. MALINDO\",\\n    year = \"2010\",\\n    pages = \"51--57\"\\n    address = \"Jakarta, Indonesia\"\\n}\\n\\n@article{sakti-s2st-csl-2013,\\n    title = \"{A-STAR}: Toward Tranlating Asian Spoken Languages\",\\n    author = \"Sakti, Sakriani and Paul, Michael and Finch, Andrew and Sakai, Shinsuke and Thang, Tat Vu, and Kimura, Noriyuki \\n    and Hori, Chiori and Sumita, Eiichiro and Nakamura, Satoshi and Park, Jun and Wutiwiwatchai, Chai and Xu, Bo and Riza, Hammam \\n    and Arora, Karunesh and Luong, Chi Mai and Li, Haizhou\",\\n    journal = \"Special issue on Speech-to-Speech Translation, Computer Speech and Language Journal\",\\n    volume = \"27\",\\n    number =\"2\",\\n    pages = \"509--527\",\\n    year = \"2013\",\\n    publisher = \"Elsevier\"\\n}\\n', description='INDspeech_NEWS_TTS is a speech dataset for developing an Indonesian text-to-speech synthesis system. The data was developed by Advanced Telecommunication Research Institute International (ATR) Japan under the the Asian speech translation advanced research (A-STAR) project [Sakti et al., 2013].\\n', homepage='https://github.com/s-sakti/data_indsp_news_tts', license='CC-BY-NC-SA 4.0')\n",
      "\n",
      "NusantaraMetadata(script='/home/samuel/anaconda2/envs/env_nusa_exp/lib/python3.8/site-packages/nusacrowd/nusa_datasets/indspeech_news_tts/indspeech_news_tts.py', dataset_name='indspeech_news_tts', tasks={<Tasks.SPEECH_RECOGNITION: 'ASR'>}, languages=['ind'], config=NusantaraConfig(name='indspeech_news_tts_120_MOS_nusantara_sptext', version=1.0.0, data_dir=None, data_files=None, description='indspeech_news_tts nusantara_sptext schema for 120 train and MOS test task', schema='nusantara_sptext', subset_id='indspeech_news_tts_120_MOS'), is_local=False, is_nusantara_schema=True, nusantara_schema_caps='SPTEXT', is_large=False, is_resource=False, is_default=False, is_broken=False, nusantara_version='1.0.0', source_version='1.0.0', citation='@inproceedings{sakti-tts-cocosda-2008,\\n    title = \"Development of HMM-based Indonesian Speech Synthesis\",\\n    author = \"Sakti, Sakriani and Maia, Ranniery and Sakai, Shinsuke and Nakamura, Satoshi\",\\n    booktitle = \"Proc. Oriental COCOSDA\",\\n    year = \"2008\",\\n    pages = \"215--220\"\\n    address = \"Kyoto, Japan\"\\n}\\n\\n@inproceedings{sakti-tts-malindo-2010,\\n    title = \"Quality and Intelligibility Assessment of Indonesian HMM-Based Speech Synthesis System\",\\n    author = \"Sakti, Sakriani and Sakai, Shinsuke and Isotani, Ryosuke and Kawai, Hisashi and Nakamura, Satoshi\",\\n    booktitle = \"Proc. MALINDO\",\\n    year = \"2010\",\\n    pages = \"51--57\"\\n    address = \"Jakarta, Indonesia\"\\n}\\n\\n@article{sakti-s2st-csl-2013,\\n    title = \"{A-STAR}: Toward Tranlating Asian Spoken Languages\",\\n    author = \"Sakti, Sakriani and Paul, Michael and Finch, Andrew and Sakai, Shinsuke and Thang, Tat Vu, and Kimura, Noriyuki \\n    and Hori, Chiori and Sumita, Eiichiro and Nakamura, Satoshi and Park, Jun and Wutiwiwatchai, Chai and Xu, Bo and Riza, Hammam \\n    and Arora, Karunesh and Luong, Chi Mai and Li, Haizhou\",\\n    journal = \"Special issue on Speech-to-Speech Translation, Computer Speech and Language Journal\",\\n    volume = \"27\",\\n    number =\"2\",\\n    pages = \"509--527\",\\n    year = \"2013\",\\n    publisher = \"Elsevier\"\\n}\\n', description='INDspeech_NEWS_TTS is a speech dataset for developing an Indonesian text-to-speech synthesis system. The data was developed by Advanced Telecommunication Research Institute International (ATR) Japan under the the Asian speech translation advanced research (A-STAR) project [Sakti et al., 2013].\\n', homepage='https://github.com/s-sakti/data_indsp_news_tts', license='CC-BY-NC-SA 4.0')\n",
      "\n",
      "NusantaraMetadata(script='/home/samuel/anaconda2/envs/env_nusa_exp/lib/python3.8/site-packages/nusacrowd/nusa_datasets/indspeech_news_tts/indspeech_news_tts.py', dataset_name='indspeech_news_tts', tasks={<Tasks.SPEECH_RECOGNITION: 'ASR'>}, languages=['ind'], config=NusantaraConfig(name='indspeech_news_tts_ZR_ZR_nusantara_sptext', version=1.0.0, data_dir=None, data_files=None, description='indspeech_news_tts nusantara_sptext schema for ZR train and ZR test task', schema='nusantara_sptext', subset_id='indspeech_news_tts_ZR_ZR'), is_local=False, is_nusantara_schema=True, nusantara_schema_caps='SPTEXT', is_large=False, is_resource=False, is_default=False, is_broken=False, nusantara_version='1.0.0', source_version='1.0.0', citation='@inproceedings{sakti-tts-cocosda-2008,\\n    title = \"Development of HMM-based Indonesian Speech Synthesis\",\\n    author = \"Sakti, Sakriani and Maia, Ranniery and Sakai, Shinsuke and Nakamura, Satoshi\",\\n    booktitle = \"Proc. Oriental COCOSDA\",\\n    year = \"2008\",\\n    pages = \"215--220\"\\n    address = \"Kyoto, Japan\"\\n}\\n\\n@inproceedings{sakti-tts-malindo-2010,\\n    title = \"Quality and Intelligibility Assessment of Indonesian HMM-Based Speech Synthesis System\",\\n    author = \"Sakti, Sakriani and Sakai, Shinsuke and Isotani, Ryosuke and Kawai, Hisashi and Nakamura, Satoshi\",\\n    booktitle = \"Proc. MALINDO\",\\n    year = \"2010\",\\n    pages = \"51--57\"\\n    address = \"Jakarta, Indonesia\"\\n}\\n\\n@article{sakti-s2st-csl-2013,\\n    title = \"{A-STAR}: Toward Tranlating Asian Spoken Languages\",\\n    author = \"Sakti, Sakriani and Paul, Michael and Finch, Andrew and Sakai, Shinsuke and Thang, Tat Vu, and Kimura, Noriyuki \\n    and Hori, Chiori and Sumita, Eiichiro and Nakamura, Satoshi and Park, Jun and Wutiwiwatchai, Chai and Xu, Bo and Riza, Hammam \\n    and Arora, Karunesh and Luong, Chi Mai and Li, Haizhou\",\\n    journal = \"Special issue on Speech-to-Speech Translation, Computer Speech and Language Journal\",\\n    volume = \"27\",\\n    number =\"2\",\\n    pages = \"509--527\",\\n    year = \"2013\",\\n    publisher = \"Elsevier\"\\n}\\n', description='INDspeech_NEWS_TTS is a speech dataset for developing an Indonesian text-to-speech synthesis system. The data was developed by Advanced Telecommunication Research Institute International (ATR) Japan under the the Asian speech translation advanced research (A-STAR) project [Sakti et al., 2013].\\n', homepage='https://github.com/s-sakti/data_indsp_news_tts', license='CC-BY-NC-SA 4.0')\n",
      "\n",
      "NusantaraMetadata(script='/home/samuel/anaconda2/envs/env_nusa_exp/lib/python3.8/site-packages/nusacrowd/nusa_datasets/indspeech_newstra_ethnicsr/indspeech_newstra_ethnicsr.py', dataset_name='indspeech_newstra_ethnicsr', tasks={<Tasks.SPEECH_RECOGNITION: 'ASR'>}, languages=['sun', 'jav', 'btk', 'ban'], config=NusantaraConfig(name='indspeech_newstra_ethnicsr_overlap_ban_nusantara_sptext', version=1.0.0, data_dir=None, data_files=None, description='indspeech_newstra_ethnicsr nusantara_sptext schema for BALI language with overlapping dataset', schema='nusantara_sptext', subset_id='indspeech_newstra_ethnicsr_overlap'), is_local=False, is_nusantara_schema=True, nusantara_schema_caps='SPTEXT', is_large=False, is_resource=False, is_default=False, is_broken=False, nusantara_version='1.0.0', source_version='1.0.0', citation='@inproceedings{sakti-cocosda-2013,\\n    title = \"Towards Language Preservation: Design and Collection of Graphemically Balanced and Parallel Speech Corpora of {I}ndonesian Ethnic Languages\",\\n    author = \"Sakti, Sakriani and Nakamura, Satoshi\",\\n    booktitle = \"Proc. Oriental COCOSDA\",\\n    year = \"2013\",\\n    address = \"Gurgaon, India\"\\n}\\n\\n@inproceedings{sakti-sltu-2014,\\n  title = \"Recent progress in developing grapheme-based speech recognition for {I}ndonesian ethnic languages: {J}avanese, {S}undanese, {B}alinese and {B}ataks\",\\n  author = \"Sakti, Sakriani and Nakamura, Satoshi\",\\n  booktitle = \"Proc. 4th Workshop on Spoken Language Technologies for Under-Resourced Languages (SLTU 2014)\",\\n  year = \"2014\",\\n  pages = \"46--52\",\\n  address = \"St. Petersburg, Russia\"\\n}\\n\\n@inproceedings{novitasari-sltu-2020,\\n  title = \"Cross-Lingual Machine Speech Chain for {J}avanese, {S}undanese, {B}alinese, and {B}ataks Speech Recognition and Synthesis\",\\n  author = \"Novitasari, Sashi and Tjandra, Andros and Sakti, Sakriani and Nakamura, Satoshi\",\\n  booktitle = \"Proc. Joint Workshop on Spoken Language Technologies for Under-resourced languages (SLTU) and Collaboration and Computing for Under-Resourced Languages (CCURL)\",\\n  year = \"2020\",\\n  pages = \"131--138\",\\n  address = \"Marseille, France\"\\n}\\n', description='INDspeech_NEWSTRA_EthnicSR is a collection of graphemically balanced and parallel speech corpora of four major Indonesian ethnic languages: Javanese, Sundanese, Balinese, and Bataks. It was developed in 2013 by the Nara Institute of Science and Technology (NAIST, Japan) [Sakti et al., 2013]. The data has been used to develop Indonesian ethnic speech recognition in supervised learning [Sakti et al., 2014] and semi-supervised learning [Novitasari et al., 2020] based on Machine Speech Chain framework [Tjandra et al., 2020].\\n', homepage='https://github.com/s-sakti/data_indsp_newstra_ethnicsr', license='CC-BY-NC-SA 4.0')\n",
      "\n",
      "NusantaraMetadata(script='/home/samuel/anaconda2/envs/env_nusa_exp/lib/python3.8/site-packages/nusacrowd/nusa_datasets/indspeech_newstra_ethnicsr/indspeech_newstra_ethnicsr.py', dataset_name='indspeech_newstra_ethnicsr', tasks={<Tasks.SPEECH_RECOGNITION: 'ASR'>}, languages=['sun', 'jav', 'btk', 'ban'], config=NusantaraConfig(name='indspeech_newstra_ethnicsr_nooverlap_ban_nusantara_sptext', version=1.0.0, data_dir=None, data_files=None, description='indspeech_newstra_ethnicsr nusantara_sptext schema for BALI language with nooverlapping dataset', schema='nusantara_sptext', subset_id='indspeech_newstra_ethnicsr_nooverlap'), is_local=False, is_nusantara_schema=True, nusantara_schema_caps='SPTEXT', is_large=False, is_resource=False, is_default=False, is_broken=False, nusantara_version='1.0.0', source_version='1.0.0', citation='@inproceedings{sakti-cocosda-2013,\\n    title = \"Towards Language Preservation: Design and Collection of Graphemically Balanced and Parallel Speech Corpora of {I}ndonesian Ethnic Languages\",\\n    author = \"Sakti, Sakriani and Nakamura, Satoshi\",\\n    booktitle = \"Proc. Oriental COCOSDA\",\\n    year = \"2013\",\\n    address = \"Gurgaon, India\"\\n}\\n\\n@inproceedings{sakti-sltu-2014,\\n  title = \"Recent progress in developing grapheme-based speech recognition for {I}ndonesian ethnic languages: {J}avanese, {S}undanese, {B}alinese and {B}ataks\",\\n  author = \"Sakti, Sakriani and Nakamura, Satoshi\",\\n  booktitle = \"Proc. 4th Workshop on Spoken Language Technologies for Under-Resourced Languages (SLTU 2014)\",\\n  year = \"2014\",\\n  pages = \"46--52\",\\n  address = \"St. Petersburg, Russia\"\\n}\\n\\n@inproceedings{novitasari-sltu-2020,\\n  title = \"Cross-Lingual Machine Speech Chain for {J}avanese, {S}undanese, {B}alinese, and {B}ataks Speech Recognition and Synthesis\",\\n  author = \"Novitasari, Sashi and Tjandra, Andros and Sakti, Sakriani and Nakamura, Satoshi\",\\n  booktitle = \"Proc. Joint Workshop on Spoken Language Technologies for Under-resourced languages (SLTU) and Collaboration and Computing for Under-Resourced Languages (CCURL)\",\\n  year = \"2020\",\\n  pages = \"131--138\",\\n  address = \"Marseille, France\"\\n}\\n', description='INDspeech_NEWSTRA_EthnicSR is a collection of graphemically balanced and parallel speech corpora of four major Indonesian ethnic languages: Javanese, Sundanese, Balinese, and Bataks. It was developed in 2013 by the Nara Institute of Science and Technology (NAIST, Japan) [Sakti et al., 2013]. The data has been used to develop Indonesian ethnic speech recognition in supervised learning [Sakti et al., 2014] and semi-supervised learning [Novitasari et al., 2020] based on Machine Speech Chain framework [Tjandra et al., 2020].\\n', homepage='https://github.com/s-sakti/data_indsp_newstra_ethnicsr', license='CC-BY-NC-SA 4.0')\n",
      "\n",
      "NusantaraMetadata(script='/home/samuel/anaconda2/envs/env_nusa_exp/lib/python3.8/site-packages/nusacrowd/nusa_datasets/indspeech_newstra_ethnicsr/indspeech_newstra_ethnicsr.py', dataset_name='indspeech_newstra_ethnicsr', tasks={<Tasks.SPEECH_RECOGNITION: 'ASR'>}, languages=['sun', 'jav', 'btk', 'ban'], config=NusantaraConfig(name='indspeech_newstra_ethnicsr_overlap_btk_nusantara_sptext', version=1.0.0, data_dir=None, data_files=None, description='indspeech_newstra_ethnicsr nusantara_sptext schema for BATAK language with overlapping dataset', schema='nusantara_sptext', subset_id='indspeech_newstra_ethnicsr_overlap'), is_local=False, is_nusantara_schema=True, nusantara_schema_caps='SPTEXT', is_large=False, is_resource=False, is_default=False, is_broken=False, nusantara_version='1.0.0', source_version='1.0.0', citation='@inproceedings{sakti-cocosda-2013,\\n    title = \"Towards Language Preservation: Design and Collection of Graphemically Balanced and Parallel Speech Corpora of {I}ndonesian Ethnic Languages\",\\n    author = \"Sakti, Sakriani and Nakamura, Satoshi\",\\n    booktitle = \"Proc. Oriental COCOSDA\",\\n    year = \"2013\",\\n    address = \"Gurgaon, India\"\\n}\\n\\n@inproceedings{sakti-sltu-2014,\\n  title = \"Recent progress in developing grapheme-based speech recognition for {I}ndonesian ethnic languages: {J}avanese, {S}undanese, {B}alinese and {B}ataks\",\\n  author = \"Sakti, Sakriani and Nakamura, Satoshi\",\\n  booktitle = \"Proc. 4th Workshop on Spoken Language Technologies for Under-Resourced Languages (SLTU 2014)\",\\n  year = \"2014\",\\n  pages = \"46--52\",\\n  address = \"St. Petersburg, Russia\"\\n}\\n\\n@inproceedings{novitasari-sltu-2020,\\n  title = \"Cross-Lingual Machine Speech Chain for {J}avanese, {S}undanese, {B}alinese, and {B}ataks Speech Recognition and Synthesis\",\\n  author = \"Novitasari, Sashi and Tjandra, Andros and Sakti, Sakriani and Nakamura, Satoshi\",\\n  booktitle = \"Proc. Joint Workshop on Spoken Language Technologies for Under-resourced languages (SLTU) and Collaboration and Computing for Under-Resourced Languages (CCURL)\",\\n  year = \"2020\",\\n  pages = \"131--138\",\\n  address = \"Marseille, France\"\\n}\\n', description='INDspeech_NEWSTRA_EthnicSR is a collection of graphemically balanced and parallel speech corpora of four major Indonesian ethnic languages: Javanese, Sundanese, Balinese, and Bataks. It was developed in 2013 by the Nara Institute of Science and Technology (NAIST, Japan) [Sakti et al., 2013]. The data has been used to develop Indonesian ethnic speech recognition in supervised learning [Sakti et al., 2014] and semi-supervised learning [Novitasari et al., 2020] based on Machine Speech Chain framework [Tjandra et al., 2020].\\n', homepage='https://github.com/s-sakti/data_indsp_newstra_ethnicsr', license='CC-BY-NC-SA 4.0')\n",
      "\n",
      "NusantaraMetadata(script='/home/samuel/anaconda2/envs/env_nusa_exp/lib/python3.8/site-packages/nusacrowd/nusa_datasets/indspeech_newstra_ethnicsr/indspeech_newstra_ethnicsr.py', dataset_name='indspeech_newstra_ethnicsr', tasks={<Tasks.SPEECH_RECOGNITION: 'ASR'>}, languages=['sun', 'jav', 'btk', 'ban'], config=NusantaraConfig(name='indspeech_newstra_ethnicsr_nooverlap_btk_nusantara_sptext', version=1.0.0, data_dir=None, data_files=None, description='indspeech_newstra_ethnicsr nusantara_sptext schema for BATAK language with nooverlapping dataset', schema='nusantara_sptext', subset_id='indspeech_newstra_ethnicsr_nooverlap'), is_local=False, is_nusantara_schema=True, nusantara_schema_caps='SPTEXT', is_large=False, is_resource=False, is_default=False, is_broken=False, nusantara_version='1.0.0', source_version='1.0.0', citation='@inproceedings{sakti-cocosda-2013,\\n    title = \"Towards Language Preservation: Design and Collection of Graphemically Balanced and Parallel Speech Corpora of {I}ndonesian Ethnic Languages\",\\n    author = \"Sakti, Sakriani and Nakamura, Satoshi\",\\n    booktitle = \"Proc. Oriental COCOSDA\",\\n    year = \"2013\",\\n    address = \"Gurgaon, India\"\\n}\\n\\n@inproceedings{sakti-sltu-2014,\\n  title = \"Recent progress in developing grapheme-based speech recognition for {I}ndonesian ethnic languages: {J}avanese, {S}undanese, {B}alinese and {B}ataks\",\\n  author = \"Sakti, Sakriani and Nakamura, Satoshi\",\\n  booktitle = \"Proc. 4th Workshop on Spoken Language Technologies for Under-Resourced Languages (SLTU 2014)\",\\n  year = \"2014\",\\n  pages = \"46--52\",\\n  address = \"St. Petersburg, Russia\"\\n}\\n\\n@inproceedings{novitasari-sltu-2020,\\n  title = \"Cross-Lingual Machine Speech Chain for {J}avanese, {S}undanese, {B}alinese, and {B}ataks Speech Recognition and Synthesis\",\\n  author = \"Novitasari, Sashi and Tjandra, Andros and Sakti, Sakriani and Nakamura, Satoshi\",\\n  booktitle = \"Proc. Joint Workshop on Spoken Language Technologies for Under-resourced languages (SLTU) and Collaboration and Computing for Under-Resourced Languages (CCURL)\",\\n  year = \"2020\",\\n  pages = \"131--138\",\\n  address = \"Marseille, France\"\\n}\\n', description='INDspeech_NEWSTRA_EthnicSR is a collection of graphemically balanced and parallel speech corpora of four major Indonesian ethnic languages: Javanese, Sundanese, Balinese, and Bataks. It was developed in 2013 by the Nara Institute of Science and Technology (NAIST, Japan) [Sakti et al., 2013]. The data has been used to develop Indonesian ethnic speech recognition in supervised learning [Sakti et al., 2014] and semi-supervised learning [Novitasari et al., 2020] based on Machine Speech Chain framework [Tjandra et al., 2020].\\n', homepage='https://github.com/s-sakti/data_indsp_newstra_ethnicsr', license='CC-BY-NC-SA 4.0')\n",
      "\n",
      "NusantaraMetadata(script='/home/samuel/anaconda2/envs/env_nusa_exp/lib/python3.8/site-packages/nusacrowd/nusa_datasets/indspeech_newstra_ethnicsr/indspeech_newstra_ethnicsr.py', dataset_name='indspeech_newstra_ethnicsr', tasks={<Tasks.SPEECH_RECOGNITION: 'ASR'>}, languages=['sun', 'jav', 'btk', 'ban'], config=NusantaraConfig(name='indspeech_newstra_ethnicsr_overlap_jav_nusantara_sptext', version=1.0.0, data_dir=None, data_files=None, description='indspeech_newstra_ethnicsr nusantara_sptext schema for JAWA language with overlapping dataset', schema='nusantara_sptext', subset_id='indspeech_newstra_ethnicsr_overlap'), is_local=False, is_nusantara_schema=True, nusantara_schema_caps='SPTEXT', is_large=False, is_resource=False, is_default=False, is_broken=False, nusantara_version='1.0.0', source_version='1.0.0', citation='@inproceedings{sakti-cocosda-2013,\\n    title = \"Towards Language Preservation: Design and Collection of Graphemically Balanced and Parallel Speech Corpora of {I}ndonesian Ethnic Languages\",\\n    author = \"Sakti, Sakriani and Nakamura, Satoshi\",\\n    booktitle = \"Proc. Oriental COCOSDA\",\\n    year = \"2013\",\\n    address = \"Gurgaon, India\"\\n}\\n\\n@inproceedings{sakti-sltu-2014,\\n  title = \"Recent progress in developing grapheme-based speech recognition for {I}ndonesian ethnic languages: {J}avanese, {S}undanese, {B}alinese and {B}ataks\",\\n  author = \"Sakti, Sakriani and Nakamura, Satoshi\",\\n  booktitle = \"Proc. 4th Workshop on Spoken Language Technologies for Under-Resourced Languages (SLTU 2014)\",\\n  year = \"2014\",\\n  pages = \"46--52\",\\n  address = \"St. Petersburg, Russia\"\\n}\\n\\n@inproceedings{novitasari-sltu-2020,\\n  title = \"Cross-Lingual Machine Speech Chain for {J}avanese, {S}undanese, {B}alinese, and {B}ataks Speech Recognition and Synthesis\",\\n  author = \"Novitasari, Sashi and Tjandra, Andros and Sakti, Sakriani and Nakamura, Satoshi\",\\n  booktitle = \"Proc. Joint Workshop on Spoken Language Technologies for Under-resourced languages (SLTU) and Collaboration and Computing for Under-Resourced Languages (CCURL)\",\\n  year = \"2020\",\\n  pages = \"131--138\",\\n  address = \"Marseille, France\"\\n}\\n', description='INDspeech_NEWSTRA_EthnicSR is a collection of graphemically balanced and parallel speech corpora of four major Indonesian ethnic languages: Javanese, Sundanese, Balinese, and Bataks. It was developed in 2013 by the Nara Institute of Science and Technology (NAIST, Japan) [Sakti et al., 2013]. The data has been used to develop Indonesian ethnic speech recognition in supervised learning [Sakti et al., 2014] and semi-supervised learning [Novitasari et al., 2020] based on Machine Speech Chain framework [Tjandra et al., 2020].\\n', homepage='https://github.com/s-sakti/data_indsp_newstra_ethnicsr', license='CC-BY-NC-SA 4.0')\n",
      "\n",
      "NusantaraMetadata(script='/home/samuel/anaconda2/envs/env_nusa_exp/lib/python3.8/site-packages/nusacrowd/nusa_datasets/indspeech_newstra_ethnicsr/indspeech_newstra_ethnicsr.py', dataset_name='indspeech_newstra_ethnicsr', tasks={<Tasks.SPEECH_RECOGNITION: 'ASR'>}, languages=['sun', 'jav', 'btk', 'ban'], config=NusantaraConfig(name='indspeech_newstra_ethnicsr_nooverlap_jav_nusantara_sptext', version=1.0.0, data_dir=None, data_files=None, description='indspeech_newstra_ethnicsr nusantara_sptext schema for JAWA language with nooverlapping dataset', schema='nusantara_sptext', subset_id='indspeech_newstra_ethnicsr_nooverlap'), is_local=False, is_nusantara_schema=True, nusantara_schema_caps='SPTEXT', is_large=False, is_resource=False, is_default=False, is_broken=False, nusantara_version='1.0.0', source_version='1.0.0', citation='@inproceedings{sakti-cocosda-2013,\\n    title = \"Towards Language Preservation: Design and Collection of Graphemically Balanced and Parallel Speech Corpora of {I}ndonesian Ethnic Languages\",\\n    author = \"Sakti, Sakriani and Nakamura, Satoshi\",\\n    booktitle = \"Proc. Oriental COCOSDA\",\\n    year = \"2013\",\\n    address = \"Gurgaon, India\"\\n}\\n\\n@inproceedings{sakti-sltu-2014,\\n  title = \"Recent progress in developing grapheme-based speech recognition for {I}ndonesian ethnic languages: {J}avanese, {S}undanese, {B}alinese and {B}ataks\",\\n  author = \"Sakti, Sakriani and Nakamura, Satoshi\",\\n  booktitle = \"Proc. 4th Workshop on Spoken Language Technologies for Under-Resourced Languages (SLTU 2014)\",\\n  year = \"2014\",\\n  pages = \"46--52\",\\n  address = \"St. Petersburg, Russia\"\\n}\\n\\n@inproceedings{novitasari-sltu-2020,\\n  title = \"Cross-Lingual Machine Speech Chain for {J}avanese, {S}undanese, {B}alinese, and {B}ataks Speech Recognition and Synthesis\",\\n  author = \"Novitasari, Sashi and Tjandra, Andros and Sakti, Sakriani and Nakamura, Satoshi\",\\n  booktitle = \"Proc. Joint Workshop on Spoken Language Technologies for Under-resourced languages (SLTU) and Collaboration and Computing for Under-Resourced Languages (CCURL)\",\\n  year = \"2020\",\\n  pages = \"131--138\",\\n  address = \"Marseille, France\"\\n}\\n', description='INDspeech_NEWSTRA_EthnicSR is a collection of graphemically balanced and parallel speech corpora of four major Indonesian ethnic languages: Javanese, Sundanese, Balinese, and Bataks. It was developed in 2013 by the Nara Institute of Science and Technology (NAIST, Japan) [Sakti et al., 2013]. The data has been used to develop Indonesian ethnic speech recognition in supervised learning [Sakti et al., 2014] and semi-supervised learning [Novitasari et al., 2020] based on Machine Speech Chain framework [Tjandra et al., 2020].\\n', homepage='https://github.com/s-sakti/data_indsp_newstra_ethnicsr', license='CC-BY-NC-SA 4.0')\n",
      "\n",
      "NusantaraMetadata(script='/home/samuel/anaconda2/envs/env_nusa_exp/lib/python3.8/site-packages/nusacrowd/nusa_datasets/indspeech_newstra_ethnicsr/indspeech_newstra_ethnicsr.py', dataset_name='indspeech_newstra_ethnicsr', tasks={<Tasks.SPEECH_RECOGNITION: 'ASR'>}, languages=['sun', 'jav', 'btk', 'ban'], config=NusantaraConfig(name='indspeech_newstra_ethnicsr_overlap_sun_nusantara_sptext', version=1.0.0, data_dir=None, data_files=None, description='indspeech_newstra_ethnicsr nusantara_sptext schema for SUNDA language with overlapping dataset', schema='nusantara_sptext', subset_id='indspeech_newstra_ethnicsr_overlap'), is_local=False, is_nusantara_schema=True, nusantara_schema_caps='SPTEXT', is_large=False, is_resource=False, is_default=False, is_broken=False, nusantara_version='1.0.0', source_version='1.0.0', citation='@inproceedings{sakti-cocosda-2013,\\n    title = \"Towards Language Preservation: Design and Collection of Graphemically Balanced and Parallel Speech Corpora of {I}ndonesian Ethnic Languages\",\\n    author = \"Sakti, Sakriani and Nakamura, Satoshi\",\\n    booktitle = \"Proc. Oriental COCOSDA\",\\n    year = \"2013\",\\n    address = \"Gurgaon, India\"\\n}\\n\\n@inproceedings{sakti-sltu-2014,\\n  title = \"Recent progress in developing grapheme-based speech recognition for {I}ndonesian ethnic languages: {J}avanese, {S}undanese, {B}alinese and {B}ataks\",\\n  author = \"Sakti, Sakriani and Nakamura, Satoshi\",\\n  booktitle = \"Proc. 4th Workshop on Spoken Language Technologies for Under-Resourced Languages (SLTU 2014)\",\\n  year = \"2014\",\\n  pages = \"46--52\",\\n  address = \"St. Petersburg, Russia\"\\n}\\n\\n@inproceedings{novitasari-sltu-2020,\\n  title = \"Cross-Lingual Machine Speech Chain for {J}avanese, {S}undanese, {B}alinese, and {B}ataks Speech Recognition and Synthesis\",\\n  author = \"Novitasari, Sashi and Tjandra, Andros and Sakti, Sakriani and Nakamura, Satoshi\",\\n  booktitle = \"Proc. Joint Workshop on Spoken Language Technologies for Under-resourced languages (SLTU) and Collaboration and Computing for Under-Resourced Languages (CCURL)\",\\n  year = \"2020\",\\n  pages = \"131--138\",\\n  address = \"Marseille, France\"\\n}\\n', description='INDspeech_NEWSTRA_EthnicSR is a collection of graphemically balanced and parallel speech corpora of four major Indonesian ethnic languages: Javanese, Sundanese, Balinese, and Bataks. It was developed in 2013 by the Nara Institute of Science and Technology (NAIST, Japan) [Sakti et al., 2013]. The data has been used to develop Indonesian ethnic speech recognition in supervised learning [Sakti et al., 2014] and semi-supervised learning [Novitasari et al., 2020] based on Machine Speech Chain framework [Tjandra et al., 2020].\\n', homepage='https://github.com/s-sakti/data_indsp_newstra_ethnicsr', license='CC-BY-NC-SA 4.0')\n",
      "\n",
      "NusantaraMetadata(script='/home/samuel/anaconda2/envs/env_nusa_exp/lib/python3.8/site-packages/nusacrowd/nusa_datasets/indspeech_newstra_ethnicsr/indspeech_newstra_ethnicsr.py', dataset_name='indspeech_newstra_ethnicsr', tasks={<Tasks.SPEECH_RECOGNITION: 'ASR'>}, languages=['sun', 'jav', 'btk', 'ban'], config=NusantaraConfig(name='indspeech_newstra_ethnicsr_nooverlap_sun_nusantara_sptext', version=1.0.0, data_dir=None, data_files=None, description='indspeech_newstra_ethnicsr nusantara_sptext schema for SUNDA language with nooverlapping dataset', schema='nusantara_sptext', subset_id='indspeech_newstra_ethnicsr_nooverlap'), is_local=False, is_nusantara_schema=True, nusantara_schema_caps='SPTEXT', is_large=False, is_resource=False, is_default=False, is_broken=False, nusantara_version='1.0.0', source_version='1.0.0', citation='@inproceedings{sakti-cocosda-2013,\\n    title = \"Towards Language Preservation: Design and Collection of Graphemically Balanced and Parallel Speech Corpora of {I}ndonesian Ethnic Languages\",\\n    author = \"Sakti, Sakriani and Nakamura, Satoshi\",\\n    booktitle = \"Proc. Oriental COCOSDA\",\\n    year = \"2013\",\\n    address = \"Gurgaon, India\"\\n}\\n\\n@inproceedings{sakti-sltu-2014,\\n  title = \"Recent progress in developing grapheme-based speech recognition for {I}ndonesian ethnic languages: {J}avanese, {S}undanese, {B}alinese and {B}ataks\",\\n  author = \"Sakti, Sakriani and Nakamura, Satoshi\",\\n  booktitle = \"Proc. 4th Workshop on Spoken Language Technologies for Under-Resourced Languages (SLTU 2014)\",\\n  year = \"2014\",\\n  pages = \"46--52\",\\n  address = \"St. Petersburg, Russia\"\\n}\\n\\n@inproceedings{novitasari-sltu-2020,\\n  title = \"Cross-Lingual Machine Speech Chain for {J}avanese, {S}undanese, {B}alinese, and {B}ataks Speech Recognition and Synthesis\",\\n  author = \"Novitasari, Sashi and Tjandra, Andros and Sakti, Sakriani and Nakamura, Satoshi\",\\n  booktitle = \"Proc. Joint Workshop on Spoken Language Technologies for Under-resourced languages (SLTU) and Collaboration and Computing for Under-Resourced Languages (CCURL)\",\\n  year = \"2020\",\\n  pages = \"131--138\",\\n  address = \"Marseille, France\"\\n}\\n', description='INDspeech_NEWSTRA_EthnicSR is a collection of graphemically balanced and parallel speech corpora of four major Indonesian ethnic languages: Javanese, Sundanese, Balinese, and Bataks. It was developed in 2013 by the Nara Institute of Science and Technology (NAIST, Japan) [Sakti et al., 2013]. The data has been used to develop Indonesian ethnic speech recognition in supervised learning [Sakti et al., 2014] and semi-supervised learning [Novitasari et al., 2020] based on Machine Speech Chain framework [Tjandra et al., 2020].\\n', homepage='https://github.com/s-sakti/data_indsp_newstra_ethnicsr', license='CC-BY-NC-SA 4.0')\n",
      "\n",
      "NusantaraMetadata(script='/home/samuel/anaconda2/envs/env_nusa_exp/lib/python3.8/site-packages/nusacrowd/nusa_datasets/indspeech_teldialog_lvcsr/indspeech_teldialog_lvcsr.py', dataset_name='indspeech_teldialog_lvcsr', tasks={<Tasks.SPEECH_RECOGNITION: 'ASR'>}, languages=['ind'], config=NusantaraConfig(name='indspeech_teldialog_lvcsr_nusantara_sptext', version='1.0.0', data_dir=None, data_files=None, description='indspeech_teldialog_lvcsr Nusantara schema', schema='nusantara_sptext', subset_id='indspeech_teldialog_lvcsr'), is_local=False, is_nusantara_schema=True, nusantara_schema_caps='SPTEXT', is_large=False, is_resource=False, is_default=False, is_broken=False, nusantara_version='1.0.0', source_version='1.0.0', citation='@inproceedings{sakti-tcast-2008,\\n    title = \"Development of {I}ndonesian Large Vocabulary Continuous Speech Recognition System within {A-STAR} Project\",\\n    author = \"Sakti, Sakriani and Kelana, Eka and Riza, Hammam and Sakai, Shinsuke and Markov, Konstantin and Nakamura, Satoshi\",\\n    booktitle = \"Proc. IJCNLP Workshop on Technologies and Corpora for Asia-Pacific Speech Translation (TCAST)\",\\n    year = \"2008\",\\n    pages = \"19--24\"\\n    address = \"Hyderabad, India\"\\n}\\n\\n\\n@inproceedings{sakti-icslp-2004,\\n    title = \"Indonesian Speech Recognition for Hearing and Speaking Impaired People\",\\n    author = \"Sakti, Sakriani and Hutagaol, Paulus and Arman, Arry Akhmad and Nakamura, Satoshi\",\\n    booktitle = \"Proc. International Conference on Spoken Language Processing (INTERSPEECH - ICSLP)\",\\n    year = \"2004\",\\n    pages = \"1037--1040\"\\n    address = \"Jeju Island, Korea\"\\n}\\n\\n@article{sakti-s2st-csl-2013,\\n    title = \"{A-STAR}: Toward Tranlating Asian Spoken Languages\",\\n    author = \"Sakti, Sakriani and Paul, Michael and Finch, Andrew and Sakai, Shinsuke and Thang, Tat Vu, and Kimura, Noriyuki \\n    and Hori, Chiori and Sumita, Eiichiro and Nakamura, Satoshi and Park, Jun and Wutiwiwatchai, Chai and Xu, Bo and Riza, Hammam \\n    and Arora, Karunesh and Luong, Chi Mai and Li, Haizhou\",\\n    journal = \"Special issue on Speech-to-Speech Translation, Computer Speech and Language Journal\",\\n    volume = \"27\",\\n    number =\"2\",\\n    pages = \"509--527\",\\n    year = \"2013\",\\n    publisher = \"Elsevier\"\\n}\\n', description='\\nINDspeech_TELDIALOG_LVCSR is one of the first Indonesian speech datasets for large vocabulary continuous speech recognition (LVCSR) based on telephon application. R&D Division of PT Telekomunikasi Indonesia developed the data in 2005-2006, in collaboration with Advanced Telecommunication Research Institute International (ATR) Japan, as the continuation of the Asia-Pacific Telecommunity (APT) project [Sakti et al., 2004]. It has also been successfully used for developing Indonesian LVCSR in the Asian speech translation advanced research (A-STAR) project [Sakti et al., 2013].\\n', homepage='https://github.com/s-sakti/data_indsp_teldialog_lvcsr', license='CC-BY-NC-SA 4.0')\n",
      "\n",
      "NusantaraMetadata(script='/home/samuel/anaconda2/envs/env_nusa_exp/lib/python3.8/site-packages/nusacrowd/nusa_datasets/indspeech_teldialog_svcsr/indspeech_teldialog_svcsr.py', dataset_name='indspeech_teldialog_svcsr', tasks={<Tasks.SPEECH_RECOGNITION: 'ASR'>}, languages=['ind'], config=NusantaraConfig(name='indspeech_teldialog_svcsr_nusantara_sptext', version=1.0.0, data_dir=None, data_files=None, description='indspeech_teldialog_svcsr Nusantara schema', schema='nusantara_sptext', subset_id='indspeech_teldialog_svcsr'), is_local=False, is_nusantara_schema=True, nusantara_schema_caps='SPTEXT', is_large=False, is_resource=False, is_default=False, is_broken=False, nusantara_version='1.0.0', source_version='1.0.0', citation='@inproceedings{sakti-icslp-2004,\\n    title = \"Indonesian Speech Recognition for Hearing and Speaking Impaired People\",\\n    author = \"Sakti, Sakriani and Hutagaol, Paulus and Arman, Arry Akhmad and Nakamura, Satoshi\",\\n    booktitle = \"Proc. International Conference on Spoken Language Processing (INTERSPEECH - ICSLP)\",\\n    year = \"2004\",\\n    pages = \"1037--1040\"\\n    address = \"Jeju Island, Korea\"\\n}\\n', description='This is the first Indonesian speech dataset for small vocabulary continuous speech recognition (SVCSR).\\nThe data was developed by TELKOMRisTI (R&D Division, PT Telekomunikasi Indonesia) in collaboration with Advanced\\nTelecommunication Research Institute International (ATR) Japan and Bandung Institute of Technology (ITB) under the\\nAsia-Pacific Telecommunity (APT) project in 2004 [Sakti et al., 2004]. Although it was originally developed for\\na telecommunication system for hearing and speaking impaired people, it can be used for other applications,\\ni.e., automatic call centers. Furthermore, as all speakers utter the same sentences,\\nit can also be used for voice conversion tasks.\\n\\nThe text is based on a word vocabulary which is derived from some necessary dialog calls,\\nsuch as dialog calls with the 119 emergency department, 108 telephone information department,\\nand ticket reservation department. In total, it consists of 20,000 utterances (about 18 hours of speech) from the\\n70-word dialog vocabulary of 100 sentences (including single word sentences) each uttered by 200 speakers\\n(100 Females, 100 Males). The age is limited to middle age (20-40 years), but they present a wide range of spoken\\ndialects from different ethnic groups. The recording is conducted in parallel for both clean and telephone speech,\\nbut we open only the clean speech due to quality issues on telephone speech.\\nEach audio file is a single-channel 16-bit PCM WAV with a sample rate of 16000 Hz.\\nThese utterances are equally split into training and test sets with 100 speakers (50 Females, 50 Males) in each set.\\n', homepage='https://github.com/s-sakti/data_indsp_teldialog_svcsr/', license='CC-BY-NC-SA-4.0')\n",
      "\n",
      "NusantaraMetadata(script='/home/samuel/anaconda2/envs/env_nusa_exp/lib/python3.8/site-packages/nusacrowd/nusa_datasets/jv_id_asr/jv_id_asr.py', dataset_name='jv_id_asr', tasks={<Tasks.SPEECH_RECOGNITION: 'ASR'>}, languages=['jav'], config=NusantaraConfig(name='jv_id_asr_nusantara_sptext', version=1.0.0, data_dir=None, data_files=None, description='jv_id_asr Nusantara schema', schema='nusantara_sptext', subset_id='jv_id_asr'), is_local=False, is_nusantara_schema=True, nusantara_schema_caps='SPTEXT', is_large=False, is_resource=False, is_default=False, is_broken=False, nusantara_version='1.0.0', source_version='1.0.0', citation='@inproceedings{kjartansson-etal-sltu2018,\\n    title = {{Crowd-Sourced Speech Corpora for Javanese, Sundanese,  Sinhala, Nepali, and Bangladeshi Bengali}},\\n    author = {Oddur Kjartansson and Supheakmungkol Sarin and Knot Pipatsrisawat and Martin Jansche and Linne Ha},\\n    booktitle = {Proc. The 6th Intl. Workshop on Spoken Language Technologies for Under-Resourced Languages (SLTU)},\\n    year  = {2018},\\n    address = {Gurugram, India},\\n    month = aug,\\n    pages = {52--55},\\n    URL   = {http://dx.doi.org/10.21437/SLTU.2018-11},\\n  }\\n', description='This data set contains transcribed audio data for Javanese. The data set consists of wave files, and a TSV file.\\nThe file utt_spk_text.tsv contains a FileID, UserID and the transcription of audio in the file.\\nThe data set has been manually quality checked, but there might still be errors.\\nThis dataset was collected by Google in collaboration with Reykjavik University and Universitas Gadjah Mada in Indonesia.\\n', homepage='http://openslr.org/35/', license='Attribution-ShareAlike 4.0 International')\n",
      "\n",
      "NusantaraMetadata(script='/home/samuel/anaconda2/envs/env_nusa_exp/lib/python3.8/site-packages/nusacrowd/nusa_datasets/jv_id_tts/jv_id_tts.py', dataset_name='jv_id_tts', tasks={<Tasks.SPEECH_RECOGNITION: 'ASR'>}, languages=['jav'], config=NusantaraConfig(name='jv_id_tts_nusantara_sptext', version=1.0.0, data_dir=None, data_files=None, description='JV_ID_TTS Nusantara schema', schema='nusantara_sptext', subset_id='jv_id_tts'), is_local=False, is_nusantara_schema=True, nusantara_schema_caps='SPTEXT', is_large=False, is_resource=False, is_default=False, is_broken=False, nusantara_version='1.0.0', source_version='1.0.0', citation='@inproceedings{sodimana18_sltu,\\n  author={Keshan Sodimana and Pasindu {De Silva} and Supheakmungkol Sarin and Oddur Kjartansson and Martin Jansche and Knot Pipatsrisawat and Linne Ha},\\n  title={{A Step-by-Step Process for Building TTS Voices Using Open Source Data and Frameworks for Bangla, Javanese, Khmer, Nepali, Sinhala, and Sundanese}},\\n  year=2018,\\n  booktitle={Proc. 6th Workshop on Spoken Language Technologies for Under-Resourced Languages (SLTU 2018)},\\n  pages={66--70},\\n  doi={10.21437/SLTU.2018-14}\\n}\\n', description='This data set contains high-quality transcribed audio data for Javanese.\\nThe data set consists of wave files, and a TSV file.\\nThe file line_index.tsv contains a filename and the transcription of audio in the file.\\nEach filename is prepended with a speaker identification number.\\nThe data set has been manually quality checked, but there might still be errors.\\nThis dataset was collected by Google in collaboration with Gadjah Mada University in Indonesia.\\n', homepage='http://openslr.org/41/', license='See https://www.openslr.org/resources/41/LICENSE file for license information. Attribution-ShareAlike 4.0 (CC BY-SA 4.0).')\n",
      "\n",
      "NusantaraMetadata(script='/home/samuel/anaconda2/envs/env_nusa_exp/lib/python3.8/site-packages/nusacrowd/nusa_datasets/librivox_indonesia/librivox_indonesia.py', dataset_name='librivox_indonesia', tasks={<Tasks.SPEECH_RECOGNITION: 'ASR'>}, languages={'ace', 'ban', 'bug', 'sun', 'ind', 'jav', 'min'}, config=NusantaraConfig(name='librivox_indonesia_nusantara_sptext', version='1.0.0', data_dir=None, data_files=None, description='Librivox-Indonesia Nusantara schema for all languages', schema='nusantara_sptext', subset_id='librivox_indonesia'), is_local=False, is_nusantara_schema=True, nusantara_schema_caps='SPTEXT', is_large=False, is_resource=False, is_default=False, is_broken=False, nusantara_version='1.0.0', source_version='1.0.0', citation='@misc{\\n   research, \\n   title={indonesian-nlp/librivox-indonesia · datasets at hugging face}, \\n   url={https://huggingface.co/datasets/indonesian-nlp/librivox-indonesia},\\n   author={Indonesian-nlp}\\n} \\n', description=\"The LibriVox Indonesia dataset consists of MP3 audio and a corresponding text file we generated from the public domain audiobooks LibriVox. \\nWe collected only languages in Indonesia for this dataset. \\nThe original LibriVox audiobooks or sound files' duration varies from a few minutes to a few hours. \\nEach audio file in the speech dataset now lasts from a few seconds to a maximum of 20 seconds.\\nWe converted the audiobooks to speech datasets using the forced alignment software we developed. \\nIt supports multilingual, including low-resource languages, such as Acehnese, Balinese, or Minangkabau. \\nWe can also use it for other languages without additional work to train the model.\\nThe dataset currently consists of 8 hours in 7 languages from Indonesia. \\nWe will add more languages or audio files as we collect them. \\n\", homepage='https://huggingface.co/indonesian-nlp/librivox-indonesia', license='CC0')\n",
      "\n",
      "NusantaraMetadata(script='/home/samuel/anaconda2/envs/env_nusa_exp/lib/python3.8/site-packages/nusacrowd/nusa_datasets/librivox_indonesia/librivox_indonesia.py', dataset_name='librivox_indonesia', tasks={<Tasks.SPEECH_RECOGNITION: 'ASR'>}, languages={'ace', 'ban', 'bug', 'sun', 'ind', 'jav', 'min'}, config=NusantaraConfig(name='librivox_indonesia_ace_nusantara_sptext', version='1.0.0', data_dir=None, data_files=None, description='Librivox-Indonesia Nusantara schema for acehnese languages', schema='nusantara_sptext', subset_id='librivox_indonesia_ace'), is_local=False, is_nusantara_schema=True, nusantara_schema_caps='SPTEXT', is_large=False, is_resource=False, is_default=False, is_broken=False, nusantara_version='1.0.0', source_version='1.0.0', citation='@misc{\\n   research, \\n   title={indonesian-nlp/librivox-indonesia · datasets at hugging face}, \\n   url={https://huggingface.co/datasets/indonesian-nlp/librivox-indonesia},\\n   author={Indonesian-nlp}\\n} \\n', description=\"The LibriVox Indonesia dataset consists of MP3 audio and a corresponding text file we generated from the public domain audiobooks LibriVox. \\nWe collected only languages in Indonesia for this dataset. \\nThe original LibriVox audiobooks or sound files' duration varies from a few minutes to a few hours. \\nEach audio file in the speech dataset now lasts from a few seconds to a maximum of 20 seconds.\\nWe converted the audiobooks to speech datasets using the forced alignment software we developed. \\nIt supports multilingual, including low-resource languages, such as Acehnese, Balinese, or Minangkabau. \\nWe can also use it for other languages without additional work to train the model.\\nThe dataset currently consists of 8 hours in 7 languages from Indonesia. \\nWe will add more languages or audio files as we collect them. \\n\", homepage='https://huggingface.co/indonesian-nlp/librivox-indonesia', license='CC0')\n",
      "\n",
      "NusantaraMetadata(script='/home/samuel/anaconda2/envs/env_nusa_exp/lib/python3.8/site-packages/nusacrowd/nusa_datasets/librivox_indonesia/librivox_indonesia.py', dataset_name='librivox_indonesia', tasks={<Tasks.SPEECH_RECOGNITION: 'ASR'>}, languages={'ace', 'ban', 'bug', 'sun', 'ind', 'jav', 'min'}, config=NusantaraConfig(name='librivox_indonesia_ban_nusantara_sptext', version='1.0.0', data_dir=None, data_files=None, description='Librivox-Indonesia Nusantara schema for balinese languages', schema='nusantara_sptext', subset_id='librivox_indonesia_ban'), is_local=False, is_nusantara_schema=True, nusantara_schema_caps='SPTEXT', is_large=False, is_resource=False, is_default=False, is_broken=False, nusantara_version='1.0.0', source_version='1.0.0', citation='@misc{\\n   research, \\n   title={indonesian-nlp/librivox-indonesia · datasets at hugging face}, \\n   url={https://huggingface.co/datasets/indonesian-nlp/librivox-indonesia},\\n   author={Indonesian-nlp}\\n} \\n', description=\"The LibriVox Indonesia dataset consists of MP3 audio and a corresponding text file we generated from the public domain audiobooks LibriVox. \\nWe collected only languages in Indonesia for this dataset. \\nThe original LibriVox audiobooks or sound files' duration varies from a few minutes to a few hours. \\nEach audio file in the speech dataset now lasts from a few seconds to a maximum of 20 seconds.\\nWe converted the audiobooks to speech datasets using the forced alignment software we developed. \\nIt supports multilingual, including low-resource languages, such as Acehnese, Balinese, or Minangkabau. \\nWe can also use it for other languages without additional work to train the model.\\nThe dataset currently consists of 8 hours in 7 languages from Indonesia. \\nWe will add more languages or audio files as we collect them. \\n\", homepage='https://huggingface.co/indonesian-nlp/librivox-indonesia', license='CC0')\n",
      "\n",
      "NusantaraMetadata(script='/home/samuel/anaconda2/envs/env_nusa_exp/lib/python3.8/site-packages/nusacrowd/nusa_datasets/librivox_indonesia/librivox_indonesia.py', dataset_name='librivox_indonesia', tasks={<Tasks.SPEECH_RECOGNITION: 'ASR'>}, languages={'ace', 'ban', 'bug', 'sun', 'ind', 'jav', 'min'}, config=NusantaraConfig(name='librivox_indonesia_bug_nusantara_sptext', version='1.0.0', data_dir=None, data_files=None, description='Librivox-Indonesia Nusantara schema for bugisnese languages', schema='nusantara_sptext', subset_id='librivox_indonesia_bug'), is_local=False, is_nusantara_schema=True, nusantara_schema_caps='SPTEXT', is_large=False, is_resource=False, is_default=False, is_broken=False, nusantara_version='1.0.0', source_version='1.0.0', citation='@misc{\\n   research, \\n   title={indonesian-nlp/librivox-indonesia · datasets at hugging face}, \\n   url={https://huggingface.co/datasets/indonesian-nlp/librivox-indonesia},\\n   author={Indonesian-nlp}\\n} \\n', description=\"The LibriVox Indonesia dataset consists of MP3 audio and a corresponding text file we generated from the public domain audiobooks LibriVox. \\nWe collected only languages in Indonesia for this dataset. \\nThe original LibriVox audiobooks or sound files' duration varies from a few minutes to a few hours. \\nEach audio file in the speech dataset now lasts from a few seconds to a maximum of 20 seconds.\\nWe converted the audiobooks to speech datasets using the forced alignment software we developed. \\nIt supports multilingual, including low-resource languages, such as Acehnese, Balinese, or Minangkabau. \\nWe can also use it for other languages without additional work to train the model.\\nThe dataset currently consists of 8 hours in 7 languages from Indonesia. \\nWe will add more languages or audio files as we collect them. \\n\", homepage='https://huggingface.co/indonesian-nlp/librivox-indonesia', license='CC0')\n",
      "\n",
      "NusantaraMetadata(script='/home/samuel/anaconda2/envs/env_nusa_exp/lib/python3.8/site-packages/nusacrowd/nusa_datasets/librivox_indonesia/librivox_indonesia.py', dataset_name='librivox_indonesia', tasks={<Tasks.SPEECH_RECOGNITION: 'ASR'>}, languages={'ace', 'ban', 'bug', 'sun', 'ind', 'jav', 'min'}, config=NusantaraConfig(name='librivox_indonesia_sun_nusantara_sptext', version='1.0.0', data_dir=None, data_files=None, description='Librivox-Indonesia Nusantara schema for sundanese languages', schema='nusantara_sptext', subset_id='librivox_indonesia_sun'), is_local=False, is_nusantara_schema=True, nusantara_schema_caps='SPTEXT', is_large=False, is_resource=False, is_default=False, is_broken=False, nusantara_version='1.0.0', source_version='1.0.0', citation='@misc{\\n   research, \\n   title={indonesian-nlp/librivox-indonesia · datasets at hugging face}, \\n   url={https://huggingface.co/datasets/indonesian-nlp/librivox-indonesia},\\n   author={Indonesian-nlp}\\n} \\n', description=\"The LibriVox Indonesia dataset consists of MP3 audio and a corresponding text file we generated from the public domain audiobooks LibriVox. \\nWe collected only languages in Indonesia for this dataset. \\nThe original LibriVox audiobooks or sound files' duration varies from a few minutes to a few hours. \\nEach audio file in the speech dataset now lasts from a few seconds to a maximum of 20 seconds.\\nWe converted the audiobooks to speech datasets using the forced alignment software we developed. \\nIt supports multilingual, including low-resource languages, such as Acehnese, Balinese, or Minangkabau. \\nWe can also use it for other languages without additional work to train the model.\\nThe dataset currently consists of 8 hours in 7 languages from Indonesia. \\nWe will add more languages or audio files as we collect them. \\n\", homepage='https://huggingface.co/indonesian-nlp/librivox-indonesia', license='CC0')\n",
      "\n",
      "NusantaraMetadata(script='/home/samuel/anaconda2/envs/env_nusa_exp/lib/python3.8/site-packages/nusacrowd/nusa_datasets/librivox_indonesia/librivox_indonesia.py', dataset_name='librivox_indonesia', tasks={<Tasks.SPEECH_RECOGNITION: 'ASR'>}, languages={'ace', 'ban', 'bug', 'sun', 'ind', 'jav', 'min'}, config=NusantaraConfig(name='librivox_indonesia_ind_nusantara_sptext', version='1.0.0', data_dir=None, data_files=None, description='Librivox-Indonesia Nusantara schema for indonesian languages', schema='nusantara_sptext', subset_id='librivox_indonesia_ind'), is_local=False, is_nusantara_schema=True, nusantara_schema_caps='SPTEXT', is_large=False, is_resource=False, is_default=False, is_broken=False, nusantara_version='1.0.0', source_version='1.0.0', citation='@misc{\\n   research, \\n   title={indonesian-nlp/librivox-indonesia · datasets at hugging face}, \\n   url={https://huggingface.co/datasets/indonesian-nlp/librivox-indonesia},\\n   author={Indonesian-nlp}\\n} \\n', description=\"The LibriVox Indonesia dataset consists of MP3 audio and a corresponding text file we generated from the public domain audiobooks LibriVox. \\nWe collected only languages in Indonesia for this dataset. \\nThe original LibriVox audiobooks or sound files' duration varies from a few minutes to a few hours. \\nEach audio file in the speech dataset now lasts from a few seconds to a maximum of 20 seconds.\\nWe converted the audiobooks to speech datasets using the forced alignment software we developed. \\nIt supports multilingual, including low-resource languages, such as Acehnese, Balinese, or Minangkabau. \\nWe can also use it for other languages without additional work to train the model.\\nThe dataset currently consists of 8 hours in 7 languages from Indonesia. \\nWe will add more languages or audio files as we collect them. \\n\", homepage='https://huggingface.co/indonesian-nlp/librivox-indonesia', license='CC0')\n",
      "\n",
      "NusantaraMetadata(script='/home/samuel/anaconda2/envs/env_nusa_exp/lib/python3.8/site-packages/nusacrowd/nusa_datasets/librivox_indonesia/librivox_indonesia.py', dataset_name='librivox_indonesia', tasks={<Tasks.SPEECH_RECOGNITION: 'ASR'>}, languages={'ace', 'ban', 'bug', 'sun', 'ind', 'jav', 'min'}, config=NusantaraConfig(name='librivox_indonesia_jav_nusantara_sptext', version='1.0.0', data_dir=None, data_files=None, description='Librivox-Indonesia Nusantara schema for javanese languages', schema='nusantara_sptext', subset_id='librivox_indonesia_jav'), is_local=False, is_nusantara_schema=True, nusantara_schema_caps='SPTEXT', is_large=False, is_resource=False, is_default=False, is_broken=False, nusantara_version='1.0.0', source_version='1.0.0', citation='@misc{\\n   research, \\n   title={indonesian-nlp/librivox-indonesia · datasets at hugging face}, \\n   url={https://huggingface.co/datasets/indonesian-nlp/librivox-indonesia},\\n   author={Indonesian-nlp}\\n} \\n', description=\"The LibriVox Indonesia dataset consists of MP3 audio and a corresponding text file we generated from the public domain audiobooks LibriVox. \\nWe collected only languages in Indonesia for this dataset. \\nThe original LibriVox audiobooks or sound files' duration varies from a few minutes to a few hours. \\nEach audio file in the speech dataset now lasts from a few seconds to a maximum of 20 seconds.\\nWe converted the audiobooks to speech datasets using the forced alignment software we developed. \\nIt supports multilingual, including low-resource languages, such as Acehnese, Balinese, or Minangkabau. \\nWe can also use it for other languages without additional work to train the model.\\nThe dataset currently consists of 8 hours in 7 languages from Indonesia. \\nWe will add more languages or audio files as we collect them. \\n\", homepage='https://huggingface.co/indonesian-nlp/librivox-indonesia', license='CC0')\n",
      "\n",
      "NusantaraMetadata(script='/home/samuel/anaconda2/envs/env_nusa_exp/lib/python3.8/site-packages/nusacrowd/nusa_datasets/librivox_indonesia/librivox_indonesia.py', dataset_name='librivox_indonesia', tasks={<Tasks.SPEECH_RECOGNITION: 'ASR'>}, languages={'ace', 'ban', 'bug', 'sun', 'ind', 'jav', 'min'}, config=NusantaraConfig(name='librivox_indonesia_min_nusantara_sptext', version='1.0.0', data_dir=None, data_files=None, description='Librivox-Indonesia Nusantara schema for minangkabau languages', schema='nusantara_sptext', subset_id='librivox_indonesia_min'), is_local=False, is_nusantara_schema=True, nusantara_schema_caps='SPTEXT', is_large=False, is_resource=False, is_default=False, is_broken=False, nusantara_version='1.0.0', source_version='1.0.0', citation='@misc{\\n   research, \\n   title={indonesian-nlp/librivox-indonesia · datasets at hugging face}, \\n   url={https://huggingface.co/datasets/indonesian-nlp/librivox-indonesia},\\n   author={Indonesian-nlp}\\n} \\n', description=\"The LibriVox Indonesia dataset consists of MP3 audio and a corresponding text file we generated from the public domain audiobooks LibriVox. \\nWe collected only languages in Indonesia for this dataset. \\nThe original LibriVox audiobooks or sound files' duration varies from a few minutes to a few hours. \\nEach audio file in the speech dataset now lasts from a few seconds to a maximum of 20 seconds.\\nWe converted the audiobooks to speech datasets using the forced alignment software we developed. \\nIt supports multilingual, including low-resource languages, such as Acehnese, Balinese, or Minangkabau. \\nWe can also use it for other languages without additional work to train the model.\\nThe dataset currently consists of 8 hours in 7 languages from Indonesia. \\nWe will add more languages or audio files as we collect them. \\n\", homepage='https://huggingface.co/indonesian-nlp/librivox-indonesia', license='CC0')\n",
      "\n",
      "NusantaraMetadata(script='/home/samuel/anaconda2/envs/env_nusa_exp/lib/python3.8/site-packages/nusacrowd/nusa_datasets/su_id_asr/su_id_asr.py', dataset_name='su_id_asr', tasks={<Tasks.SPEECH_RECOGNITION: 'ASR'>}, languages=['sun'], config=NusantaraConfig(name='su_id_asr_nusantara_sptext', version=1.0.0, data_dir=None, data_files=None, description='SU_ID_ASR Nusantara schema', schema='nusantara_sptext', subset_id='su_id_asr'), is_local=False, is_nusantara_schema=True, nusantara_schema_caps='SPTEXT', is_large=False, is_resource=False, is_default=False, is_broken=False, nusantara_version='1.0.0', source_version='1.0.0', citation='@inproceedings{sodimana18_sltu,\\n  author={Keshan Sodimana and Pasindu {De Silva} and Supheakmungkol Sarin and Oddur Kjartansson and Martin Jansche and Knot Pipatsrisawat and Linne Ha},\\n  title={{A Step-by-Step Process for Building TTS Voices Using Open Source Data and Frameworks for Bangla, Javanese, Khmer, Nepali, Sinhala, and Sundanese}},\\n  year=2018,\\n  booktitle={Proc. 6th Workshop on Spoken Language Technologies for Under-Resourced Languages (SLTU 2018)},\\n  pages={66--70},\\n  doi={10.21437/SLTU.2018-14}\\n}\\n', description='Sundanese ASR training data set containing ~220K utterances.\\nThis dataset was collected by Google in Indonesia.\\n\\n\\n', homepage='https://indonlp.github.io/nusa-catalogue/card.html?su_id_asr', license='Attribution-ShareAlike 4.0 International.')\n",
      "\n",
      "NusantaraMetadata(script='/home/samuel/anaconda2/envs/env_nusa_exp/lib/python3.8/site-packages/nusacrowd/nusa_datasets/su_id_tts/su_id_tts.py', dataset_name='su_id_tts', tasks={<Tasks.SPEECH_RECOGNITION: 'ASR'>}, languages=['sun'], config=NusantaraConfig(name='su_id_tts_nusantara_sptext', version=1.0.0, data_dir=None, data_files=None, description='SU_ID_TTS Nusantara schema', schema='nusantara_sptext', subset_id='su_id_tts'), is_local=False, is_nusantara_schema=True, nusantara_schema_caps='SPTEXT', is_large=False, is_resource=False, is_default=False, is_broken=False, nusantara_version='1.0.0', source_version='1.0.0', citation='@inproceedings{sodimana18_sltu,\\n  author={Keshan Sodimana and Pasindu {De Silva} and Supheakmungkol Sarin and Oddur Kjartansson and Martin Jansche and Knot Pipatsrisawat and Linne Ha},\\n  title={{A Step-by-Step Process for Building TTS Voices Using Open Source Data and Frameworks for Bangla, Javanese, Khmer, Nepali, Sinhala, and Sundanese}},\\n  year=2018,\\n  booktitle={Proc. 6th Workshop on Spoken Language Technologies for Under-Resourced Languages (SLTU 2018)},\\n  pages={66--70},\\n  doi={10.21437/SLTU.2018-14}\\n}\\n', description='This data set contains high-quality transcribed audio data for Sundanese. The data set consists of wave files, and a TSV file. The file line_index.tsv contains a filename and the transcription of audio in the file. Each filename is prepended with a speaker identification number.\\nThe data set has been manually quality checked, but there might still be errors.\\nThis dataset was collected by Google in collaboration with Universitas Pendidikan Indonesia.\\n', homepage='http://openslr.org/44/', license='CC BY-SA 4.0')\n",
      "\n",
      "NusantaraMetadata(script='/home/samuel/anaconda2/envs/env_nusa_exp/lib/python3.8/site-packages/nusacrowd/nusa_datasets/titml_idn/titml_idn.py', dataset_name='titml_idn', tasks={<Tasks.SPEECH_RECOGNITION: 'ASR'>}, languages=['ind'], config=NusantaraConfig(name='titml_idn_nusantara_sptext', version=1.0.0, data_dir=None, data_files=None, description='TITML-IDN Nusantara schema', schema='nusantara_sptext', subset_id='titml_idn'), is_local=False, is_nusantara_schema=True, nusantara_schema_caps='SPTEXT', is_large=False, is_resource=False, is_default=False, is_broken=False, nusantara_version='1.0.0', source_version='1.0.0', citation='@inproceedings{lestari2006titmlidn,\\n  title={A large vocabulary continuous speech recognition system for Indonesian language},\\n  author={Lestari, Dessi Puji and Iwano, Koji and Furui, Sadaoki},\\n  booktitle={15th Indonesian Scientific Conference in Japan Proceedings},\\n  pages={17--22},\\n  year={2006}\\n}\\n', description='TITML-IDN (Tokyo Institute of Technology Multilingual - Indonesian) is collected to build a pioneering Indonesian Large Vocabulary Continuous Speech Recognition (LVCSR) System. In order to build an LVCSR system, high accurate acoustic models and large-scale language models are essential. Since Indonesian speech corpus was not available yet, we tried to collect speech data from 20 Indonesian native speakers (11 males and 9 females) to construct a speech corpus for training the acoustic model based on Hidden Markov Models (HMMs). A text corpus which was collected by ILPS, Informatics Institute, University of Amsterdam, was used to build a 40K-vocabulary dictionary and a n-gram language model.\\n', homepage='http://research.nii.ac.jp/src/en/TITML-IDN.html', license='For research purposes only. If you use this corpus, you have to cite (Lestari et al, 2006).')\n"
     ]
    }
   ],
   "source": [
    "conhelps = NusantaraConfigHelper()\n",
    "\n",
    "speech_tasks = [\n",
    "    Tasks.SPEECH_RECOGNITION, Tasks.SPEECH_TO_TEXT_TRANSLATION,\n",
    "    Tasks.TEXT_TO_SPEECH,Tasks.SPEECH_TO_SPEECH_TRANSLATION\n",
    "]\n",
    "\n",
    "# nusantara NER public tasks\n",
    "print('Nusantara Speech public datasets')\n",
    "nc_speech_public_helpers = conhelps.filtered(\n",
    "    lambda x: (\n",
    "        x.is_nusantara_schema\n",
    "        and any([x in speech_tasks for x in x.tasks])\n",
    "        and not x.is_local\n",
    "    )\n",
    ")\n",
    "print(nc_speech_public_helpers)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6c319bc0-9130-41b5-9915-bb91b7ae7e97",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['cvss_c_nusantara_s2s', 'cvss_t_nusantara_s2s']"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "[x.config.name for x in conhelps.filtered(\n",
    "    lambda x: (\n",
    "        x.is_nusantara_schema\n",
    "        and 'cvss' in x.dataset_name\n",
    "        and not x.is_local\n",
    "    )\n",
    ")]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "74e0aa7d-458b-41fb-b150-35791bb4285e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['covost2',\n",
       " 'covost2',\n",
       " 'cvss',\n",
       " 'cvss',\n",
       " 'indspeech_digit_cdsr',\n",
       " 'indspeech_news_ethnicsr',\n",
       " 'indspeech_news_ethnicsr',\n",
       " 'indspeech_news_ethnicsr',\n",
       " 'indspeech_news_ethnicsr',\n",
       " 'indspeech_news_lvcsr',\n",
       " 'indspeech_newstra_ethnicsr',\n",
       " 'indspeech_newstra_ethnicsr',\n",
       " 'indspeech_newstra_ethnicsr',\n",
       " 'indspeech_newstra_ethnicsr',\n",
       " 'indspeech_newstra_ethnicsr',\n",
       " 'indspeech_newstra_ethnicsr',\n",
       " 'indspeech_newstra_ethnicsr',\n",
       " 'indspeech_newstra_ethnicsr',\n",
       " 'indspeech_teldialog_lvcsr',\n",
       " 'indspeech_teldialog_svcsr',\n",
       " 'jv_id_asr',\n",
       " 'jv_id_tts',\n",
       " 'librivox_indonesia',\n",
       " 'librivox_indonesia',\n",
       " 'librivox_indonesia',\n",
       " 'librivox_indonesia',\n",
       " 'librivox_indonesia',\n",
       " 'librivox_indonesia',\n",
       " 'librivox_indonesia',\n",
       " 'librivox_indonesia',\n",
       " 'su_id_asr',\n",
       " 'su_id_tts',\n",
       " 'titml_idn']"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "[x.dataset_name for x in nc_speech_public_helpers]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "ba151cb3-5fb9-43e7-b267-febbd7c3e9cf",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['covost2_ind_eng_nusantara_sptext',\n",
       " 'covost2_eng_ind_nusantara_sptext',\n",
       " 'cvss_c_nusantara_s2s',\n",
       " 'cvss_t_nusantara_s2s',\n",
       " 'indspeech_digit_cdsr_nusantara_sptext',\n",
       " 'indspeech_news_ethnicsr_jv_overlap_nusantara_sptext',\n",
       " 'indspeech_news_ethnicsr_su_overlap_nusantara_sptext',\n",
       " 'indspeech_news_ethnicsr_jv_nooverlap_nusantara_sptext',\n",
       " 'indspeech_news_ethnicsr_su_nooverlap_nusantara_sptext',\n",
       " 'indspeech_news_lvcsr_nusantara_sptext',\n",
       " 'indspeech_newstra_ethnicsr_overlap_ban_nusantara_sptext',\n",
       " 'indspeech_newstra_ethnicsr_nooverlap_ban_nusantara_sptext',\n",
       " 'indspeech_newstra_ethnicsr_overlap_btk_nusantara_sptext',\n",
       " 'indspeech_newstra_ethnicsr_nooverlap_btk_nusantara_sptext',\n",
       " 'indspeech_newstra_ethnicsr_overlap_jav_nusantara_sptext',\n",
       " 'indspeech_newstra_ethnicsr_nooverlap_jav_nusantara_sptext',\n",
       " 'indspeech_newstra_ethnicsr_overlap_sun_nusantara_sptext',\n",
       " 'indspeech_newstra_ethnicsr_nooverlap_sun_nusantara_sptext',\n",
       " 'indspeech_teldialog_lvcsr_nusantara_sptext',\n",
       " 'indspeech_teldialog_svcsr_nusantara_sptext',\n",
       " 'jv_id_asr_nusantara_sptext',\n",
       " 'jv_id_tts_nusantara_sptext',\n",
       " 'librivox_indonesia_nusantara_sptext',\n",
       " 'librivox_indonesia_ace_nusantara_sptext',\n",
       " 'librivox_indonesia_bug_nusantara_sptext',\n",
       " 'librivox_indonesia_sun_nusantara_sptext',\n",
       " 'librivox_indonesia_ban_nusantara_sptext',\n",
       " 'librivox_indonesia_min_nusantara_sptext',\n",
       " 'librivox_indonesia_ind_nusantara_sptext',\n",
       " 'librivox_indonesia_jav_nusantara_sptext',\n",
       " 'su_id_asr_nusantara_sptext',\n",
       " 'su_id_tts_nusantara_sptext',\n",
       " 'titml_idn_nusantara_sptext']"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "[x.config.name for x in nc_speech_public_helpers]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "30648d98-791b-4330-a9d0-16afc1f0d722",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "38"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len([x.config.name for x in nc_speech_public_helpers])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d55195cb-c91d-4edb-8423-338901002e09",
   "metadata": {},
   "outputs": [],
   "source": [
    "metadata_helper = NusantaraMetadataHelper()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "142a4212-ee66-4f70-9679-ef456737e34f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<nusantara.config_helper.NusantaraMetadataHelper at 0x7f7ad20432b0>"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "metadata_helper"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "b65e3b7c-0879-4ba9-b1c5-2447e9bf7f09",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['bible_en_id_nusantara_t2t', 'bible_jv_id_nusantara_t2t', 'bible_su_id_nusantara_t2t', 'covost2_ind_eng_nusantara_t2t', 'covost2_eng_ind_nusantara_t2t', 'id_panl_bppt_nusantara_t2t', 'id_qqp_nusantara_t2t', 'id_wiki_parallel_jav_ind_nusantara_t2t', 'id_wiki_parallel_min_ind_nusantara_t2t', 'id_wiki_parallel_sun_ind_nusantara_t2t', 'indo_general_mt_en_id_nusantara_t2t', 'indo_religious_mt_en_id_nusantara_t2t', 'indosum_fold0_nusantara_t2t', 'indosum_fold1_nusantara_t2t', 'indosum_fold2_nusantara_t2t', 'indosum_fold3_nusantara_t2t', 'indosum_fold4_nusantara_t2t', 'korpus_nusantara_ind_jav_nusantara_t2t', 'korpus_nusantara_ind_day_nusantara_t2t', 'korpus_nusantara_ind_bug_nusantara_t2t', 'korpus_nusantara_ind_sun_nusantara_t2t', 'korpus_nusantara_ind_mad_nusantara_t2t', 'korpus_nusantara_ind_bin_nusantara_t2t', 'korpus_nusantara_ind_bbc_nusantara_t2t', 'korpus_nusantara_ind_khek_nusantara_t2t', 'korpus_nusantara_ind_msa_nusantara_t2t', 'korpus_nusantara_ind_min_nusantara_t2t', 'korpus_nusantara_ind_tiociu_nusantara_t2t', 'korpus_nusantara_jav_ind_nusantara_t2t', 'korpus_nusantara_day_ind_nusantara_t2t', 'korpus_nusantara_bug_ind_nusantara_t2t', 'korpus_nusantara_sun_ind_nusantara_t2t', 'korpus_nusantara_mad_ind_nusantara_t2t', 'korpus_nusantara_bin_ind_nusantara_t2t', 'korpus_nusantara_bbc_ind_nusantara_t2t', 'korpus_nusantara_khek_ind_nusantara_t2t', 'korpus_nusantara_msa_ind_nusantara_t2t', 'korpus_nusantara_min_ind_nusantara_t2t', 'korpus_nusantara_tiociu_ind_nusantara_t2t', 'minangnlp_mt_nusantara_t2t', 'multilexnorm_nusantara_t2t', 'news_en_id_nusantara_t2t', 'nllb_seed_ace_nusantara_t2t', 'nllb_seed_bjn_nusantara_t2t', 'nllb_seed_bug_nusantara_t2t', 'nusax_mt_ace_ind_nusantara_t2t', 'nusax_mt_ban_ind_nusantara_t2t', 'nusax_mt_bjn_ind_nusantara_t2t', 'nusax_mt_bug_ind_nusantara_t2t', 'nusax_mt_eng_ind_nusantara_t2t', 'nusax_mt_ind_ace_nusantara_t2t', 'nusax_mt_ind_ban_nusantara_t2t', 'nusax_mt_ind_bjn_nusantara_t2t', 'nusax_mt_ind_bug_nusantara_t2t', 'nusax_mt_ind_eng_nusantara_t2t', 'nusax_mt_ind_jav_nusantara_t2t', 'nusax_mt_ind_mad_nusantara_t2t', 'nusax_mt_ind_min_nusantara_t2t', 'nusax_mt_ind_nij_nusantara_t2t', 'nusax_mt_ind_sun_nusantara_t2t', 'nusax_mt_ind_bbc_nusantara_t2t', 'nusax_mt_jav_ind_nusantara_t2t', 'nusax_mt_mad_ind_nusantara_t2t', 'nusax_mt_min_ind_nusantara_t2t', 'nusax_mt_nij_ind_nusantara_t2t', 'nusax_mt_sun_ind_nusantara_t2t', 'nusax_mt_bbc_ind_nusantara_t2t', 'paracotta_id_nusantara_t2t', 'parallel_su_id_nusantara_t2t', 'stif_indonesia_nusantara_t2t', 'talpco_eng_ind_nusantara_t2t', 'talpco_eng_jpn_nusantara_t2t', 'talpco_eng_kor_nusantara_t2t', 'talpco_eng_myn_nusantara_t2t', 'talpco_eng_tha_nusantara_t2t', 'talpco_eng_vie_nusantara_t2t', 'talpco_eng_zsm_nusantara_t2t', 'talpco_ind_eng_nusantara_t2t', 'talpco_ind_jpn_nusantara_t2t', 'talpco_ind_kor_nusantara_t2t', 'talpco_ind_myn_nusantara_t2t', 'talpco_ind_tha_nusantara_t2t', 'talpco_ind_vie_nusantara_t2t', 'talpco_ind_zsm_nusantara_t2t', 'talpco_jpn_eng_nusantara_t2t', 'talpco_jpn_ind_nusantara_t2t', 'talpco_jpn_kor_nusantara_t2t', 'talpco_jpn_myn_nusantara_t2t', 'talpco_jpn_tha_nusantara_t2t', 'talpco_jpn_vie_nusantara_t2t', 'talpco_jpn_zsm_nusantara_t2t', 'talpco_kor_eng_nusantara_t2t', 'talpco_kor_ind_nusantara_t2t', 'talpco_kor_jpn_nusantara_t2t', 'talpco_kor_myn_nusantara_t2t', 'talpco_kor_tha_nusantara_t2t', 'talpco_kor_vie_nusantara_t2t', 'talpco_kor_zsm_nusantara_t2t', 'talpco_myn_eng_nusantara_t2t', 'talpco_myn_ind_nusantara_t2t', 'talpco_myn_jpn_nusantara_t2t', 'talpco_myn_kor_nusantara_t2t', 'talpco_myn_tha_nusantara_t2t', 'talpco_myn_vie_nusantara_t2t', 'talpco_myn_zsm_nusantara_t2t', 'talpco_tha_eng_nusantara_t2t', 'talpco_tha_ind_nusantara_t2t', 'talpco_tha_jpn_nusantara_t2t', 'talpco_tha_kor_nusantara_t2t', 'talpco_tha_myn_nusantara_t2t', 'talpco_tha_vie_nusantara_t2t', 'talpco_tha_zsm_nusantara_t2t', 'talpco_vie_eng_nusantara_t2t', 'talpco_vie_ind_nusantara_t2t', 'talpco_vie_jpn_nusantara_t2t', 'talpco_vie_kor_nusantara_t2t', 'talpco_vie_myn_nusantara_t2t', 'talpco_vie_tha_nusantara_t2t', 'talpco_vie_zsm_nusantara_t2t', 'talpco_zsm_eng_nusantara_t2t', 'talpco_zsm_ind_nusantara_t2t', 'talpco_zsm_jpn_nusantara_t2t', 'talpco_zsm_kor_nusantara_t2t', 'talpco_zsm_myn_nusantara_t2t', 'talpco_zsm_tha_nusantara_t2t', 'talpco_zsm_vie_nusantara_t2t', 'talpco_nusantara_t2t', 'ted_en_id_nusantara_t2t', 'ud_id_csui_nusantara_t2t', 'xl_sum_nusantara_t2t', 'xpersona_id_nusantara_t2t']\n"
     ]
    }
   ],
   "source": [
    "print([helper.config.name for helper in conhelps.filtered(lambda x: (\n",
    "    't2t' in x.config.name\n",
    "    and x.is_nusantara_schema\n",
    "    and ('ind' in x.config.name if 'nusax_mt' in x.dataset_name else True)\n",
    "    and not x.is_resource))\n",
    "])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "816f260a-6f20-41ab-a000-790884061878",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['casa_nusantara_text_multi', 'emot_nusantara_text', 'emotcmt_nusantara_text', 'emotion_id_opinion_nusantara_text', 'hoasa_nusantara_text_multi', 'id_abusive_nusantara_text', 'id_abusive_news_comment_nusantara_text', 'id_clickbait_nusantara_text', 'id_google_play_review_nusantara_text', 'id_google_play_review_posneg_nusantara_text', 'id_hatespeech_nusantara_text', 'id_hoax_news_nusantara_text', 'id_multilabel_hs_nusantara_text_multi', 'id_short_answer_grading_nusantara_pairs', 'id_stance_nusantara_pairs', 'id_sts_nusantara_pairs_score', 'imdb_jv_nusantara_text', 'indolem_ntp_nusantara_pairs', 'indolem_sentiment_nusantara_text', 'indonli_nusantara_pairs', 'jadi_ide_nusantara_text', 'local_id_abusive_jav_nusantara_text_multi', 'local_id_abusive_sun_nusantara_text_multi', 'netifier_nusantara_text_multi', 'nusax_senti_ace_nusantara_text', 'nusax_senti_ban_nusantara_text', 'nusax_senti_bjn_nusantara_text', 'nusax_senti_bug_nusantara_text', 'nusax_senti_eng_nusantara_text', 'nusax_senti_ind_nusantara_text', 'nusax_senti_jav_nusantara_text', 'nusax_senti_mad_nusantara_text', 'nusax_senti_min_nusantara_text', 'nusax_senti_nij_nusantara_text', 'nusax_senti_sun_nusantara_text', 'nusax_senti_bbc_nusantara_text', 'nusax_senti_nusantara_text', 'sentiment_nathasa_review_nusantara_text', 'smsa_nusantara_text', 'wrete_nusantara_pairs']\n"
     ]
    }
   ],
   "source": [
    "print([helper.config.name for helper in conhelps.filtered(lambda x: (\n",
    "    ('_text' in x.config.name or '_pair' in x.config.name)\n",
    "    and x.is_nusantara_schema\n",
    "    and not x.is_resource))\n",
    "])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "9fb5fd8d-fcdc-4203-b704-094aae284aac",
   "metadata": {},
   "outputs": [],
   "source": [
    "@dataclass\n",
    "class MetaDict:\n",
    "    data: dict = None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "id": "d14640d3-6f18-49f9-a026-ca57982bb9a0",
   "metadata": {},
   "outputs": [],
   "source": [
    "meta_df = pd.read_csv('https://docs.google.com/spreadsheets/d/17o83IvWxmtGLYridZis0nEprHhsZIMeFtHGtXV35h6M/export?format=csv&gid=879729812', skiprows=1)\n",
    "meta_df = meta_df[meta_df['Implemented'] != 0].rename({\n",
    "    'No.': 'id', 'Name': 'name', 'Subsets': 'subsets', 'Link': 'source_link', 'Description': 'description',\n",
    "    'HF Link': 'hf_link', 'License': 'license', 'Year': 'year', 'Collection Style': 'collection_style',\n",
    "    'Language': 'language', 'Dialect': 'dialect', 'Domain': 'domain', 'Form': 'modality', 'Tasks': 'tasks',\n",
    "    'Volume': 'volume', 'Unit': 'unit', 'Ethical Risks': 'ethical_risk', 'Provider': 'provider',\n",
    "    'Paper Title': 'paper_title', 'Paper Link': 'paper_link', 'Access': 'access', 'Derived From': 'derived_from', \n",
    "    'Test Split': 'is_splitted', 'Notes': 'notes', 'Dataloader': 'dataloader', 'Implemented': 'implemented'\n",
    "}, axis=1)\n",
    "meta_df['is_splitted'] = meta_df['is_splitted'].apply(lambda x: True if x =='Yes' else False)\n",
    "# [\n",
    "#  'No.', 'Name', 'Subsets', 'Link', 'HF Link', 'License', 'Year',\n",
    "#  'Language', 'Dialect', 'Domain', 'Form', 'Collection Style',\n",
    "#  'Description', 'Volume', 'Unit', 'Ethical Risks', 'Provider',\n",
    "#  'Paper Title', 'Paper Link', 'Access', 'Derived From', 'Tasks',\n",
    "#  'Test Split', 'Notes', 'Dataloader', 'Implemented'\n",
    "# ]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "ce44a6d9-0b40-49f9-9b2d-5c91b69a0cc0",
   "metadata": {},
   "outputs": [],
   "source": [
    "name_to_meta_map = {}\n",
    "for cfg_meta in conhelps:\n",
    "    # Assign metadata to meta dataframe\n",
    "    meta_df.loc[meta_df.dataloader == cfg_meta.dataset_name, [\n",
    "        'is_large', 'is_resource', 'is_default', 'is_broken',\n",
    "        'is_local', 'citation', 'license', 'homepage', 'tasks'\n",
    "    ]] = [\n",
    "        cfg_meta.is_large, cfg_meta.is_resource, cfg_meta.is_default, cfg_meta.is_broken, \n",
    "        cfg_meta.is_local, cfg_meta.citation, cfg_meta.license, cfg_meta.homepage, '|'.join([task.value for task in cfg_meta.tasks])\n",
    "    ]\n",
    "    \n",
    "    if cfg_meta.dataset_name not in name_to_meta_map:\n",
    "        name_to_meta_map[cfg_meta.dataset_name] = {}\n",
    "    if cfg_meta.config.schema not in name_to_meta_map[cfg_meta.dataset_name]:\n",
    "        name_to_meta_map[cfg_meta.dataset_name][cfg_meta.config.schema] = []\n",
    "    name_to_meta_map[cfg_meta.dataset_name][cfg_meta.config.schema].append(cfg_meta)\n",
    "    \n",
    "meta_df = meta_df.fillna(False)\n",
    "\n",
    "for dset_name in name_to_meta_map.keys():\n",
    "    meta_df.loc[meta_df.dataloader == dset_name, 'metadata'] = MetaDict(data=name_to_meta_map[dset_name])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "04571177-ca67-4e86-a1a9-1ae0fc24d9ab",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:datasets.builder:Reusing dataset sentiment_nathasa_review (/home/samuel/.cache/huggingface/datasets/sentiment_nathasa_review/sentiment_nathasa_review_nusantara_text/1.0.0/30eb0ea3a17ed2a9baea98645648c8f32804db498c2d57a7e376d733f0f9b89f)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6f6fd52667d044c9a337745549c43793",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/2 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:datasets.builder:Reusing dataset indolem_sentiment_dataset (/home/samuel/.cache/huggingface/datasets/indolem_sentiment_dataset/indolem_sentiment_nusantara_text/1.0.0/b9050e6fcb7b7fa40f4dd502c92dc8c1df0ef30e067f1664af76c908c7b47467)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2312b38fdd2c4a1ba9e6a80e162695e0",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/3 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:datasets.builder:Reusing dataset nusa_x_senti (/home/samuel/.cache/huggingface/datasets/nusa_x_senti/nusax_senti_ind_nusantara_text/1.0.0/3c834957d94e799c678f9be42ee6def4fcdbfdd0407b6add0d1bc875067b2ff9)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "93465d742c864a2e92651c197fe987ec",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/3 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:datasets.builder:Reusing dataset smsa (/home/samuel/.cache/huggingface/datasets/smsa/smsa_nusantara_text/1.0.0/c4914239fa9f7683736a2df989690905e1551838621298d799f0d1c78097a349)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b4eac36d0daa4c1e81330f4c39ab9163",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/3 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "{'sentiment_nathasa_review_nusantara_text': DatasetDict({\n",
       "     train: Dataset({\n",
       "         features: ['id', 'text', 'label'],\n",
       "         num_rows: 62132\n",
       "     })\n",
       "     test: Dataset({\n",
       "         features: ['id', 'text', 'label'],\n",
       "         num_rows: 62131\n",
       "     })\n",
       " }),\n",
       " 'indolem_sentiment_nusantara_text': DatasetDict({\n",
       "     train: Dataset({\n",
       "         features: ['id', 'text', 'label'],\n",
       "         num_rows: 3638\n",
       "     })\n",
       "     test: Dataset({\n",
       "         features: ['id', 'text', 'label'],\n",
       "         num_rows: 1011\n",
       "     })\n",
       "     validation: Dataset({\n",
       "         features: ['id', 'text', 'label'],\n",
       "         num_rows: 399\n",
       "     })\n",
       " }),\n",
       " 'nusax_senti_ind_nusantara_text': DatasetDict({\n",
       "     train: Dataset({\n",
       "         features: ['id', 'text', 'label'],\n",
       "         num_rows: 500\n",
       "     })\n",
       "     validation: Dataset({\n",
       "         features: ['id', 'text', 'label'],\n",
       "         num_rows: 100\n",
       "     })\n",
       "     test: Dataset({\n",
       "         features: ['id', 'text', 'label'],\n",
       "         num_rows: 400\n",
       "     })\n",
       " }),\n",
       " 'smsa_nusantara_text': DatasetDict({\n",
       "     train: Dataset({\n",
       "         features: ['id', 'text', 'label'],\n",
       "         num_rows: 11000\n",
       "     })\n",
       "     validation: Dataset({\n",
       "         features: ['id', 'text', 'label'],\n",
       "         num_rows: 1260\n",
       "     })\n",
       "     test: Dataset({\n",
       "         features: ['id', 'text', 'label'],\n",
       "         num_rows: 500\n",
       "     })\n",
       " })}"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Filter & load all Indonesian sentiment analysis task\n",
    "lang = 'ind'\n",
    "task = Tasks.SENTIMENT_ANALYSIS\n",
    "\n",
    "filtered_df = meta_df.loc[\n",
    "    (meta_df.tasks.str.contains(task.value)) & (~meta_df.is_resource) &\n",
    "    (meta_df.language.str.contains(lang)) & meta_df.is_splitted\n",
    "]\n",
    "\n",
    "schema = f'nusantara_{TASK_TO_SCHEMA[task].lower()}'\n",
    "datasets = {}\n",
    "for metas in filtered_df.metadata:\n",
    "    if schema in metas.data:\n",
    "        for meta in metas.data[schema]:\n",
    "            if len(meta.languages) > 1:\n",
    "                if lang in meta.config.name:\n",
    "                    datasets[meta.config.name] = meta.load_dataset()\n",
    "            else:\n",
    "                datasets[meta.config.name] = meta.load_dataset()\n",
    "datasets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "02fafbb2-0a92-44f8-abb2-c2c1bd95264b",
   "metadata": {
    "scrolled": true,
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:datasets.builder:Reusing dataset indo_general_mt_en_id (/home/samuel/.cache/huggingface/datasets/indo_general_mt_en_id/indo_general_mt_en_id_nusantara_t2t/1.0.0/8997ff3f81a5107662a8a4d0e405a1e053a076327735146c76b56dd6c263e4c1)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3bfb28684dfd4db6842abd239f70a0c3",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/3 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:datasets.builder:Reusing dataset nusa_xmt (/home/samuel/.cache/huggingface/datasets/nusa_xmt/nusax_mt_ace_eng_nusantara_t2t/1.0.0/8e5cd18bf2771dbfe991f0c4db45678ec7aaecf94420cd375f211b46495e81cf)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7f9aebd180ae4f41b0008ce41618b5a4",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/3 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:datasets.builder:Reusing dataset nusa_xmt (/home/samuel/.cache/huggingface/datasets/nusa_xmt/nusax_mt_ban_eng_nusantara_t2t/1.0.0/8e5cd18bf2771dbfe991f0c4db45678ec7aaecf94420cd375f211b46495e81cf)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "63199a6569984ab1bc06cbf9f8f0c7cf",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/3 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:datasets.builder:Reusing dataset nusa_xmt (/home/samuel/.cache/huggingface/datasets/nusa_xmt/nusax_mt_bjn_eng_nusantara_t2t/1.0.0/8e5cd18bf2771dbfe991f0c4db45678ec7aaecf94420cd375f211b46495e81cf)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "de148d27485f4423930374041840e681",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/3 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:datasets.builder:Reusing dataset nusa_xmt (/home/samuel/.cache/huggingface/datasets/nusa_xmt/nusax_mt_bug_eng_nusantara_t2t/1.0.0/8e5cd18bf2771dbfe991f0c4db45678ec7aaecf94420cd375f211b46495e81cf)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "733cf0d06bfd45f08de4e8743806284b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/3 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:datasets.builder:Reusing dataset nusa_xmt (/home/samuel/.cache/huggingface/datasets/nusa_xmt/nusax_mt_eng_ace_nusantara_t2t/1.0.0/8e5cd18bf2771dbfe991f0c4db45678ec7aaecf94420cd375f211b46495e81cf)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "50b42fa9d6374b4e882fe035161f010f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/3 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:datasets.builder:Reusing dataset nusa_xmt (/home/samuel/.cache/huggingface/datasets/nusa_xmt/nusax_mt_eng_ban_nusantara_t2t/1.0.0/8e5cd18bf2771dbfe991f0c4db45678ec7aaecf94420cd375f211b46495e81cf)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a1e7c4acc7f241f2ab024821e2b14426",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/3 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:datasets.builder:Reusing dataset nusa_xmt (/home/samuel/.cache/huggingface/datasets/nusa_xmt/nusax_mt_eng_bjn_nusantara_t2t/1.0.0/8e5cd18bf2771dbfe991f0c4db45678ec7aaecf94420cd375f211b46495e81cf)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "16b03766aa2a46b0a213a3f30fa5f077",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/3 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:datasets.builder:Reusing dataset nusa_xmt (/home/samuel/.cache/huggingface/datasets/nusa_xmt/nusax_mt_eng_bug_nusantara_t2t/1.0.0/8e5cd18bf2771dbfe991f0c4db45678ec7aaecf94420cd375f211b46495e81cf)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "aa2ce1e8a96e45b5964b5479b8d4e0ae",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/3 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:datasets.builder:Reusing dataset nusa_xmt (/home/samuel/.cache/huggingface/datasets/nusa_xmt/nusax_mt_eng_ind_nusantara_t2t/1.0.0/8e5cd18bf2771dbfe991f0c4db45678ec7aaecf94420cd375f211b46495e81cf)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "180c9747baac4f28a3ee7eecb2103716",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/3 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:datasets.builder:Reusing dataset nusa_xmt (/home/samuel/.cache/huggingface/datasets/nusa_xmt/nusax_mt_eng_jav_nusantara_t2t/1.0.0/8e5cd18bf2771dbfe991f0c4db45678ec7aaecf94420cd375f211b46495e81cf)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "eac7f667242c4e89a876b3467ac58d58",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/3 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:datasets.builder:Reusing dataset nusa_xmt (/home/samuel/.cache/huggingface/datasets/nusa_xmt/nusax_mt_eng_mad_nusantara_t2t/1.0.0/8e5cd18bf2771dbfe991f0c4db45678ec7aaecf94420cd375f211b46495e81cf)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "70212851f5a34594a9eff799d9fe5a04",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/3 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:datasets.builder:Reusing dataset nusa_xmt (/home/samuel/.cache/huggingface/datasets/nusa_xmt/nusax_mt_eng_min_nusantara_t2t/1.0.0/8e5cd18bf2771dbfe991f0c4db45678ec7aaecf94420cd375f211b46495e81cf)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "867f508faad447149af027519cce3352",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/3 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:datasets.builder:Reusing dataset nusa_xmt (/home/samuel/.cache/huggingface/datasets/nusa_xmt/nusax_mt_eng_nij_nusantara_t2t/1.0.0/8e5cd18bf2771dbfe991f0c4db45678ec7aaecf94420cd375f211b46495e81cf)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "adb4e7bf54d94e7e9220f22ccfef8442",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/3 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:datasets.builder:Reusing dataset nusa_xmt (/home/samuel/.cache/huggingface/datasets/nusa_xmt/nusax_mt_eng_sun_nusantara_t2t/1.0.0/8e5cd18bf2771dbfe991f0c4db45678ec7aaecf94420cd375f211b46495e81cf)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c3698a06451c4eaa8c81badfc406a945",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/3 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:datasets.builder:Reusing dataset nusa_xmt (/home/samuel/.cache/huggingface/datasets/nusa_xmt/nusax_mt_eng_bbc_nusantara_t2t/1.0.0/8e5cd18bf2771dbfe991f0c4db45678ec7aaecf94420cd375f211b46495e81cf)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "fa1bd9e8ac60459b91c4bb0277d8fda0",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/3 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:datasets.builder:Reusing dataset nusa_xmt (/home/samuel/.cache/huggingface/datasets/nusa_xmt/nusax_mt_ind_eng_nusantara_t2t/1.0.0/8e5cd18bf2771dbfe991f0c4db45678ec7aaecf94420cd375f211b46495e81cf)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a22f53a080bd45f2bccad78ff3938668",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/3 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:datasets.builder:Reusing dataset nusa_xmt (/home/samuel/.cache/huggingface/datasets/nusa_xmt/nusax_mt_jav_eng_nusantara_t2t/1.0.0/8e5cd18bf2771dbfe991f0c4db45678ec7aaecf94420cd375f211b46495e81cf)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b3e9f6cfa3894da8933f63d0339f30c2",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/3 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:datasets.builder:Reusing dataset nusa_xmt (/home/samuel/.cache/huggingface/datasets/nusa_xmt/nusax_mt_mad_eng_nusantara_t2t/1.0.0/8e5cd18bf2771dbfe991f0c4db45678ec7aaecf94420cd375f211b46495e81cf)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "79e5c74f8f7742ac8fb9a326e393b5ea",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/3 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:datasets.builder:Reusing dataset nusa_xmt (/home/samuel/.cache/huggingface/datasets/nusa_xmt/nusax_mt_min_eng_nusantara_t2t/1.0.0/8e5cd18bf2771dbfe991f0c4db45678ec7aaecf94420cd375f211b46495e81cf)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "fcc4a94a0ac0459c8edab8fef21c4f5f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/3 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:datasets.builder:Reusing dataset nusa_xmt (/home/samuel/.cache/huggingface/datasets/nusa_xmt/nusax_mt_nij_eng_nusantara_t2t/1.0.0/8e5cd18bf2771dbfe991f0c4db45678ec7aaecf94420cd375f211b46495e81cf)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5041c936a409414bb0cfbf15bc1b34ba",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/3 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:datasets.builder:Reusing dataset nusa_xmt (/home/samuel/.cache/huggingface/datasets/nusa_xmt/nusax_mt_sun_eng_nusantara_t2t/1.0.0/8e5cd18bf2771dbfe991f0c4db45678ec7aaecf94420cd375f211b46495e81cf)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "73a35f6f3b974311855fbb68ec610846",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/3 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:datasets.builder:Reusing dataset nusa_xmt (/home/samuel/.cache/huggingface/datasets/nusa_xmt/nusax_mt_bbc_eng_nusantara_t2t/1.0.0/8e5cd18bf2771dbfe991f0c4db45678ec7aaecf94420cd375f211b46495e81cf)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6a17186c0f264f7ba0392e8f338228b5",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/3 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "{'indo_general_mt_en_id_nusantara_t2t': DatasetDict({\n",
       "     train: Dataset({\n",
       "         features: ['id', 'text_1', 'text_2', 'text_1_name', 'text_2_name'],\n",
       "         num_rows: 1821716\n",
       "     })\n",
       "     test: Dataset({\n",
       "         features: ['id', 'text_1', 'text_2', 'text_1_name', 'text_2_name'],\n",
       "         num_rows: 2000\n",
       "     })\n",
       "     validation: Dataset({\n",
       "         features: ['id', 'text_1', 'text_2', 'text_1_name', 'text_2_name'],\n",
       "         num_rows: 2000\n",
       "     })\n",
       " }),\n",
       " 'nusax_mt_ace_eng_nusantara_t2t': DatasetDict({\n",
       "     train: Dataset({\n",
       "         features: ['id', 'text_1', 'text_2', 'text_1_name', 'text_2_name'],\n",
       "         num_rows: 500\n",
       "     })\n",
       "     validation: Dataset({\n",
       "         features: ['id', 'text_1', 'text_2', 'text_1_name', 'text_2_name'],\n",
       "         num_rows: 100\n",
       "     })\n",
       "     test: Dataset({\n",
       "         features: ['id', 'text_1', 'text_2', 'text_1_name', 'text_2_name'],\n",
       "         num_rows: 400\n",
       "     })\n",
       " }),\n",
       " 'nusax_mt_ban_eng_nusantara_t2t': DatasetDict({\n",
       "     train: Dataset({\n",
       "         features: ['id', 'text_1', 'text_2', 'text_1_name', 'text_2_name'],\n",
       "         num_rows: 500\n",
       "     })\n",
       "     validation: Dataset({\n",
       "         features: ['id', 'text_1', 'text_2', 'text_1_name', 'text_2_name'],\n",
       "         num_rows: 100\n",
       "     })\n",
       "     test: Dataset({\n",
       "         features: ['id', 'text_1', 'text_2', 'text_1_name', 'text_2_name'],\n",
       "         num_rows: 400\n",
       "     })\n",
       " }),\n",
       " 'nusax_mt_bjn_eng_nusantara_t2t': DatasetDict({\n",
       "     train: Dataset({\n",
       "         features: ['id', 'text_1', 'text_2', 'text_1_name', 'text_2_name'],\n",
       "         num_rows: 500\n",
       "     })\n",
       "     validation: Dataset({\n",
       "         features: ['id', 'text_1', 'text_2', 'text_1_name', 'text_2_name'],\n",
       "         num_rows: 100\n",
       "     })\n",
       "     test: Dataset({\n",
       "         features: ['id', 'text_1', 'text_2', 'text_1_name', 'text_2_name'],\n",
       "         num_rows: 400\n",
       "     })\n",
       " }),\n",
       " 'nusax_mt_bug_eng_nusantara_t2t': DatasetDict({\n",
       "     train: Dataset({\n",
       "         features: ['id', 'text_1', 'text_2', 'text_1_name', 'text_2_name'],\n",
       "         num_rows: 500\n",
       "     })\n",
       "     validation: Dataset({\n",
       "         features: ['id', 'text_1', 'text_2', 'text_1_name', 'text_2_name'],\n",
       "         num_rows: 100\n",
       "     })\n",
       "     test: Dataset({\n",
       "         features: ['id', 'text_1', 'text_2', 'text_1_name', 'text_2_name'],\n",
       "         num_rows: 400\n",
       "     })\n",
       " }),\n",
       " 'nusax_mt_eng_ace_nusantara_t2t': DatasetDict({\n",
       "     train: Dataset({\n",
       "         features: ['id', 'text_1', 'text_2', 'text_1_name', 'text_2_name'],\n",
       "         num_rows: 500\n",
       "     })\n",
       "     validation: Dataset({\n",
       "         features: ['id', 'text_1', 'text_2', 'text_1_name', 'text_2_name'],\n",
       "         num_rows: 100\n",
       "     })\n",
       "     test: Dataset({\n",
       "         features: ['id', 'text_1', 'text_2', 'text_1_name', 'text_2_name'],\n",
       "         num_rows: 400\n",
       "     })\n",
       " }),\n",
       " 'nusax_mt_eng_ban_nusantara_t2t': DatasetDict({\n",
       "     train: Dataset({\n",
       "         features: ['id', 'text_1', 'text_2', 'text_1_name', 'text_2_name'],\n",
       "         num_rows: 500\n",
       "     })\n",
       "     validation: Dataset({\n",
       "         features: ['id', 'text_1', 'text_2', 'text_1_name', 'text_2_name'],\n",
       "         num_rows: 100\n",
       "     })\n",
       "     test: Dataset({\n",
       "         features: ['id', 'text_1', 'text_2', 'text_1_name', 'text_2_name'],\n",
       "         num_rows: 400\n",
       "     })\n",
       " }),\n",
       " 'nusax_mt_eng_bjn_nusantara_t2t': DatasetDict({\n",
       "     train: Dataset({\n",
       "         features: ['id', 'text_1', 'text_2', 'text_1_name', 'text_2_name'],\n",
       "         num_rows: 500\n",
       "     })\n",
       "     validation: Dataset({\n",
       "         features: ['id', 'text_1', 'text_2', 'text_1_name', 'text_2_name'],\n",
       "         num_rows: 100\n",
       "     })\n",
       "     test: Dataset({\n",
       "         features: ['id', 'text_1', 'text_2', 'text_1_name', 'text_2_name'],\n",
       "         num_rows: 400\n",
       "     })\n",
       " }),\n",
       " 'nusax_mt_eng_bug_nusantara_t2t': DatasetDict({\n",
       "     train: Dataset({\n",
       "         features: ['id', 'text_1', 'text_2', 'text_1_name', 'text_2_name'],\n",
       "         num_rows: 500\n",
       "     })\n",
       "     validation: Dataset({\n",
       "         features: ['id', 'text_1', 'text_2', 'text_1_name', 'text_2_name'],\n",
       "         num_rows: 100\n",
       "     })\n",
       "     test: Dataset({\n",
       "         features: ['id', 'text_1', 'text_2', 'text_1_name', 'text_2_name'],\n",
       "         num_rows: 400\n",
       "     })\n",
       " }),\n",
       " 'nusax_mt_eng_ind_nusantara_t2t': DatasetDict({\n",
       "     train: Dataset({\n",
       "         features: ['id', 'text_1', 'text_2', 'text_1_name', 'text_2_name'],\n",
       "         num_rows: 500\n",
       "     })\n",
       "     validation: Dataset({\n",
       "         features: ['id', 'text_1', 'text_2', 'text_1_name', 'text_2_name'],\n",
       "         num_rows: 100\n",
       "     })\n",
       "     test: Dataset({\n",
       "         features: ['id', 'text_1', 'text_2', 'text_1_name', 'text_2_name'],\n",
       "         num_rows: 400\n",
       "     })\n",
       " }),\n",
       " 'nusax_mt_eng_jav_nusantara_t2t': DatasetDict({\n",
       "     train: Dataset({\n",
       "         features: ['id', 'text_1', 'text_2', 'text_1_name', 'text_2_name'],\n",
       "         num_rows: 500\n",
       "     })\n",
       "     validation: Dataset({\n",
       "         features: ['id', 'text_1', 'text_2', 'text_1_name', 'text_2_name'],\n",
       "         num_rows: 100\n",
       "     })\n",
       "     test: Dataset({\n",
       "         features: ['id', 'text_1', 'text_2', 'text_1_name', 'text_2_name'],\n",
       "         num_rows: 400\n",
       "     })\n",
       " }),\n",
       " 'nusax_mt_eng_mad_nusantara_t2t': DatasetDict({\n",
       "     train: Dataset({\n",
       "         features: ['id', 'text_1', 'text_2', 'text_1_name', 'text_2_name'],\n",
       "         num_rows: 500\n",
       "     })\n",
       "     validation: Dataset({\n",
       "         features: ['id', 'text_1', 'text_2', 'text_1_name', 'text_2_name'],\n",
       "         num_rows: 100\n",
       "     })\n",
       "     test: Dataset({\n",
       "         features: ['id', 'text_1', 'text_2', 'text_1_name', 'text_2_name'],\n",
       "         num_rows: 400\n",
       "     })\n",
       " }),\n",
       " 'nusax_mt_eng_min_nusantara_t2t': DatasetDict({\n",
       "     train: Dataset({\n",
       "         features: ['id', 'text_1', 'text_2', 'text_1_name', 'text_2_name'],\n",
       "         num_rows: 500\n",
       "     })\n",
       "     validation: Dataset({\n",
       "         features: ['id', 'text_1', 'text_2', 'text_1_name', 'text_2_name'],\n",
       "         num_rows: 100\n",
       "     })\n",
       "     test: Dataset({\n",
       "         features: ['id', 'text_1', 'text_2', 'text_1_name', 'text_2_name'],\n",
       "         num_rows: 400\n",
       "     })\n",
       " }),\n",
       " 'nusax_mt_eng_nij_nusantara_t2t': DatasetDict({\n",
       "     train: Dataset({\n",
       "         features: ['id', 'text_1', 'text_2', 'text_1_name', 'text_2_name'],\n",
       "         num_rows: 500\n",
       "     })\n",
       "     validation: Dataset({\n",
       "         features: ['id', 'text_1', 'text_2', 'text_1_name', 'text_2_name'],\n",
       "         num_rows: 100\n",
       "     })\n",
       "     test: Dataset({\n",
       "         features: ['id', 'text_1', 'text_2', 'text_1_name', 'text_2_name'],\n",
       "         num_rows: 400\n",
       "     })\n",
       " }),\n",
       " 'nusax_mt_eng_sun_nusantara_t2t': DatasetDict({\n",
       "     train: Dataset({\n",
       "         features: ['id', 'text_1', 'text_2', 'text_1_name', 'text_2_name'],\n",
       "         num_rows: 500\n",
       "     })\n",
       "     validation: Dataset({\n",
       "         features: ['id', 'text_1', 'text_2', 'text_1_name', 'text_2_name'],\n",
       "         num_rows: 100\n",
       "     })\n",
       "     test: Dataset({\n",
       "         features: ['id', 'text_1', 'text_2', 'text_1_name', 'text_2_name'],\n",
       "         num_rows: 400\n",
       "     })\n",
       " }),\n",
       " 'nusax_mt_eng_bbc_nusantara_t2t': DatasetDict({\n",
       "     train: Dataset({\n",
       "         features: ['id', 'text_1', 'text_2', 'text_1_name', 'text_2_name'],\n",
       "         num_rows: 500\n",
       "     })\n",
       "     validation: Dataset({\n",
       "         features: ['id', 'text_1', 'text_2', 'text_1_name', 'text_2_name'],\n",
       "         num_rows: 100\n",
       "     })\n",
       "     test: Dataset({\n",
       "         features: ['id', 'text_1', 'text_2', 'text_1_name', 'text_2_name'],\n",
       "         num_rows: 400\n",
       "     })\n",
       " }),\n",
       " 'nusax_mt_ind_eng_nusantara_t2t': DatasetDict({\n",
       "     train: Dataset({\n",
       "         features: ['id', 'text_1', 'text_2', 'text_1_name', 'text_2_name'],\n",
       "         num_rows: 500\n",
       "     })\n",
       "     validation: Dataset({\n",
       "         features: ['id', 'text_1', 'text_2', 'text_1_name', 'text_2_name'],\n",
       "         num_rows: 100\n",
       "     })\n",
       "     test: Dataset({\n",
       "         features: ['id', 'text_1', 'text_2', 'text_1_name', 'text_2_name'],\n",
       "         num_rows: 400\n",
       "     })\n",
       " }),\n",
       " 'nusax_mt_jav_eng_nusantara_t2t': DatasetDict({\n",
       "     train: Dataset({\n",
       "         features: ['id', 'text_1', 'text_2', 'text_1_name', 'text_2_name'],\n",
       "         num_rows: 500\n",
       "     })\n",
       "     validation: Dataset({\n",
       "         features: ['id', 'text_1', 'text_2', 'text_1_name', 'text_2_name'],\n",
       "         num_rows: 100\n",
       "     })\n",
       "     test: Dataset({\n",
       "         features: ['id', 'text_1', 'text_2', 'text_1_name', 'text_2_name'],\n",
       "         num_rows: 400\n",
       "     })\n",
       " }),\n",
       " 'nusax_mt_mad_eng_nusantara_t2t': DatasetDict({\n",
       "     train: Dataset({\n",
       "         features: ['id', 'text_1', 'text_2', 'text_1_name', 'text_2_name'],\n",
       "         num_rows: 500\n",
       "     })\n",
       "     validation: Dataset({\n",
       "         features: ['id', 'text_1', 'text_2', 'text_1_name', 'text_2_name'],\n",
       "         num_rows: 100\n",
       "     })\n",
       "     test: Dataset({\n",
       "         features: ['id', 'text_1', 'text_2', 'text_1_name', 'text_2_name'],\n",
       "         num_rows: 400\n",
       "     })\n",
       " }),\n",
       " 'nusax_mt_min_eng_nusantara_t2t': DatasetDict({\n",
       "     train: Dataset({\n",
       "         features: ['id', 'text_1', 'text_2', 'text_1_name', 'text_2_name'],\n",
       "         num_rows: 500\n",
       "     })\n",
       "     validation: Dataset({\n",
       "         features: ['id', 'text_1', 'text_2', 'text_1_name', 'text_2_name'],\n",
       "         num_rows: 100\n",
       "     })\n",
       "     test: Dataset({\n",
       "         features: ['id', 'text_1', 'text_2', 'text_1_name', 'text_2_name'],\n",
       "         num_rows: 400\n",
       "     })\n",
       " }),\n",
       " 'nusax_mt_nij_eng_nusantara_t2t': DatasetDict({\n",
       "     train: Dataset({\n",
       "         features: ['id', 'text_1', 'text_2', 'text_1_name', 'text_2_name'],\n",
       "         num_rows: 500\n",
       "     })\n",
       "     validation: Dataset({\n",
       "         features: ['id', 'text_1', 'text_2', 'text_1_name', 'text_2_name'],\n",
       "         num_rows: 100\n",
       "     })\n",
       "     test: Dataset({\n",
       "         features: ['id', 'text_1', 'text_2', 'text_1_name', 'text_2_name'],\n",
       "         num_rows: 400\n",
       "     })\n",
       " }),\n",
       " 'nusax_mt_sun_eng_nusantara_t2t': DatasetDict({\n",
       "     train: Dataset({\n",
       "         features: ['id', 'text_1', 'text_2', 'text_1_name', 'text_2_name'],\n",
       "         num_rows: 500\n",
       "     })\n",
       "     validation: Dataset({\n",
       "         features: ['id', 'text_1', 'text_2', 'text_1_name', 'text_2_name'],\n",
       "         num_rows: 100\n",
       "     })\n",
       "     test: Dataset({\n",
       "         features: ['id', 'text_1', 'text_2', 'text_1_name', 'text_2_name'],\n",
       "         num_rows: 400\n",
       "     })\n",
       " }),\n",
       " 'nusax_mt_bbc_eng_nusantara_t2t': DatasetDict({\n",
       "     train: Dataset({\n",
       "         features: ['id', 'text_1', 'text_2', 'text_1_name', 'text_2_name'],\n",
       "         num_rows: 500\n",
       "     })\n",
       "     validation: Dataset({\n",
       "         features: ['id', 'text_1', 'text_2', 'text_1_name', 'text_2_name'],\n",
       "         num_rows: 100\n",
       "     })\n",
       "     test: Dataset({\n",
       "         features: ['id', 'text_1', 'text_2', 'text_1_name', 'text_2_name'],\n",
       "         num_rows: 400\n",
       "     })\n",
       " })}"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Filter & load all splitted English sentiment analysis task\n",
    "lang = 'eng'\n",
    "task = Tasks.MACHINE_TRANSLATION\n",
    "\n",
    "filtered_df = meta_df.loc[\n",
    "    (meta_df.tasks.str.contains(task.value)) & \n",
    "    (meta_df.language.str.contains(lang)) & meta_df.is_splitted\n",
    "]\n",
    "\n",
    "schema = f'nusantara_{TASK_TO_SCHEMA[task].lower()}'\n",
    "datasets = {}\n",
    "for metas in filtered_df.metadata:\n",
    "    if schema in metas.data:\n",
    "        for meta in metas.data[schema]:\n",
    "            if len(meta.languages) > 1:\n",
    "                if lang in meta.config.name:\n",
    "                    datasets[meta.config.name] = meta.load_dataset()\n",
    "            else:\n",
    "                datasets[meta.config.name] = meta.load_dataset()\n",
    "                \n",
    "datasets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "123a219f-007f-44c0-b490-376eb94c873c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>name</th>\n",
       "      <th>subsets</th>\n",
       "      <th>source_link</th>\n",
       "      <th>hf_link</th>\n",
       "      <th>license</th>\n",
       "      <th>year</th>\n",
       "      <th>language</th>\n",
       "      <th>dialect</th>\n",
       "      <th>domain</th>\n",
       "      <th>...</th>\n",
       "      <th>dataloader</th>\n",
       "      <th>implemented</th>\n",
       "      <th>is_large</th>\n",
       "      <th>is_resource</th>\n",
       "      <th>is_default</th>\n",
       "      <th>is_broken</th>\n",
       "      <th>is_local</th>\n",
       "      <th>citation</th>\n",
       "      <th>homepage</th>\n",
       "      <th>metadata</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>27</td>\n",
       "      <td>Idn-tagged-corpus-CSUI</td>\n",
       "      <td>False</td>\n",
       "      <td>https://github.com/ir-nlp-csui/idn-tagged-corp...</td>\n",
       "      <td>https://huggingface.co/datasets/indonlu</td>\n",
       "      <td>Creative Commons Attribution Share-Alike 4.0 I...</td>\n",
       "      <td>2014.0</td>\n",
       "      <td>ind</td>\n",
       "      <td>formal</td>\n",
       "      <td>news articles</td>\n",
       "      <td>...</td>\n",
       "      <td>idn_tagged_corpus_csui</td>\n",
       "      <td>17.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>@inproceedings{dinakaramani2014designing,\\n  t...</td>\n",
       "      <td>https://bahasa.cs.ui.ac.id/postag/corpus</td>\n",
       "      <td>MetaDict(data={'source': [NusantaraMetadata(sc...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>153</th>\n",
       "      <td>116</td>\n",
       "      <td>UD_Indonesian-CSUI</td>\n",
       "      <td>False</td>\n",
       "      <td>https://github.com/UniversalDependencies/UD_In...</td>\n",
       "      <td>False</td>\n",
       "      <td>CC BY-SA 4.0</td>\n",
       "      <td>2020.0</td>\n",
       "      <td>ind</td>\n",
       "      <td>False</td>\n",
       "      <td>General, News articles</td>\n",
       "      <td>...</td>\n",
       "      <td>ud_id_csui</td>\n",
       "      <td>1.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>@article {10.3844/jcssp.2020.1585.1597,\\nautho...</td>\n",
       "      <td>https://github.com/UniversalDependencies/UD_In...</td>\n",
       "      <td>MetaDict(data={'source': [NusantaraMetadata(sc...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2 rows × 34 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      id                    name subsets  \\\n",
       "29    27  Idn-tagged-corpus-CSUI   False   \n",
       "153  116      UD_Indonesian-CSUI   False   \n",
       "\n",
       "                                           source_link  \\\n",
       "29   https://github.com/ir-nlp-csui/idn-tagged-corp...   \n",
       "153  https://github.com/UniversalDependencies/UD_In...   \n",
       "\n",
       "                                     hf_link  \\\n",
       "29   https://huggingface.co/datasets/indonlu   \n",
       "153                                    False   \n",
       "\n",
       "                                               license    year language  \\\n",
       "29   Creative Commons Attribution Share-Alike 4.0 I...  2014.0      ind   \n",
       "153                                       CC BY-SA 4.0  2020.0      ind   \n",
       "\n",
       "    dialect                  domain  ...              dataloader implemented  \\\n",
       "29   formal           news articles  ...  idn_tagged_corpus_csui        17.0   \n",
       "153   False  General, News articles  ...              ud_id_csui         1.0   \n",
       "\n",
       "    is_large is_resource is_default is_broken is_local  \\\n",
       "29     False       False      False     False    False   \n",
       "153    False       False      False     False    False   \n",
       "\n",
       "                                              citation  \\\n",
       "29   @inproceedings{dinakaramani2014designing,\\n  t...   \n",
       "153  @article {10.3844/jcssp.2020.1585.1597,\\nautho...   \n",
       "\n",
       "                                              homepage  \\\n",
       "29            https://bahasa.cs.ui.ac.id/postag/corpus   \n",
       "153  https://github.com/UniversalDependencies/UD_In...   \n",
       "\n",
       "                                              metadata  \n",
       "29   MetaDict(data={'source': [NusantaraMetadata(sc...  \n",
       "153  MetaDict(data={'source': [NusantaraMetadata(sc...  \n",
       "\n",
       "[2 rows x 34 columns]"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "meta_df.loc[\n",
    "    (meta_df.name.str.contains('CSUI'))\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "c8ef7b46-2598-40ac-9cbc-61ff2f64d85e",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:datasets.builder:Reusing dataset sentiment_nathasa_review (/home/samuel/.cache/huggingface/datasets/sentiment_nathasa_review/sentiment_nathasa_review_nusantara_text/1.0.0/30eb0ea3a17ed2a9baea98645648c8f32804db498c2d57a7e376d733f0f9b89f)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a848485ef6ad4644b74393ba0a273b45",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/2 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:datasets.builder:Reusing dataset indolem_sentiment_dataset (/home/samuel/.cache/huggingface/datasets/indolem_sentiment_dataset/indolem_sentiment_nusantara_text/1.0.0/b9050e6fcb7b7fa40f4dd502c92dc8c1df0ef30e067f1664af76c908c7b47467)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "fe55d1e564e641c2b23d652d16fe39c5",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/3 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:datasets.builder:Reusing dataset inset_lexicon (/home/samuel/.cache/huggingface/datasets/inset_lexicon/inset_lexicon_nusantara_text/1.0.0/337c7878219884751038f73a138360c5973ad6a454289b1d5e0572220d443fa0)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "02c9b52e51cc4779b7df2518ef8a54df",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/1 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:datasets.builder:Reusing dataset nusa_x_senti (/home/samuel/.cache/huggingface/datasets/nusa_x_senti/nusax_senti_ind_nusantara_text/1.0.0/3c834957d94e799c678f9be42ee6def4fcdbfdd0407b6add0d1bc875067b2ff9)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6333ac0c7a2040eeb38cc70a81e9db19",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/3 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:datasets.builder:Reusing dataset smsa (/home/samuel/.cache/huggingface/datasets/smsa/smsa_nusantara_text/1.0.0/c4914239fa9f7683736a2df989690905e1551838621298d799f0d1c78097a349)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1e8a58d41cef468895e5385fa5cb7025",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/3 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "{'sentiment_nathasa_review_nusantara_text': DatasetDict({\n",
       "     train: Dataset({\n",
       "         features: ['id', 'text', 'label'],\n",
       "         num_rows: 62132\n",
       "     })\n",
       "     test: Dataset({\n",
       "         features: ['id', 'text', 'label'],\n",
       "         num_rows: 62131\n",
       "     })\n",
       " }),\n",
       " 'indolem_sentiment_nusantara_text': DatasetDict({\n",
       "     train: Dataset({\n",
       "         features: ['id', 'text', 'label'],\n",
       "         num_rows: 3638\n",
       "     })\n",
       "     test: Dataset({\n",
       "         features: ['id', 'text', 'label'],\n",
       "         num_rows: 1011\n",
       "     })\n",
       "     validation: Dataset({\n",
       "         features: ['id', 'text', 'label'],\n",
       "         num_rows: 399\n",
       "     })\n",
       " }),\n",
       " 'inset_lexicon_nusantara_text': DatasetDict({\n",
       "     train: Dataset({\n",
       "         features: ['id', 'text', 'label'],\n",
       "         num_rows: 10218\n",
       "     })\n",
       " }),\n",
       " 'nusax_senti_ind_nusantara_text': DatasetDict({\n",
       "     train: Dataset({\n",
       "         features: ['id', 'text', 'label'],\n",
       "         num_rows: 500\n",
       "     })\n",
       "     validation: Dataset({\n",
       "         features: ['id', 'text', 'label'],\n",
       "         num_rows: 100\n",
       "     })\n",
       "     test: Dataset({\n",
       "         features: ['id', 'text', 'label'],\n",
       "         num_rows: 400\n",
       "     })\n",
       " }),\n",
       " 'smsa_nusantara_text': DatasetDict({\n",
       "     train: Dataset({\n",
       "         features: ['id', 'text', 'label'],\n",
       "         num_rows: 11000\n",
       "     })\n",
       "     validation: Dataset({\n",
       "         features: ['id', 'text', 'label'],\n",
       "         num_rows: 1260\n",
       "     })\n",
       "     test: Dataset({\n",
       "         features: ['id', 'text', 'label'],\n",
       "         num_rows: 500\n",
       "     })\n",
       " })}"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Filter & load all Indonesian sentiment analysis task\n",
    "lang = 'ind'\n",
    "task = Tasks.SENTIMENT_ANALYSIS\n",
    "\n",
    "filtered_df = meta_df.loc[\n",
    "    (meta_df.tasks.str.contains(task.value)) & \n",
    "    (meta_df.language.str.contains(lang)) & meta_df.is_splitted\n",
    "]\n",
    "\n",
    "schema = f'nusantara_{TASK_TO_SCHEMA[task].lower()}'\n",
    "datasets = {}\n",
    "for metas in filtered_df.metadata:\n",
    "    if schema in metas.data:\n",
    "        for meta in metas.data[schema]:\n",
    "            if len(meta.languages) > 1:\n",
    "                if lang in meta.config.name:\n",
    "                    datasets[meta.config.name] = meta.load_dataset()\n",
    "            else:\n",
    "                datasets[meta.config.name] = meta.load_dataset()\n",
    "\n",
    "datasets"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3958ab4b-3567-4170-aaa6-4fa3bd8da47d",
   "metadata": {},
   "source": [
    "If there is multiple tasks on a dataset, split it and filter schema out of it resulting in\n",
    "config meta from the original dataset meta"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "630f8fb1-840b-48e8-84ad-bfc0fa43a9a9",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>name</th>\n",
       "      <th>subsets</th>\n",
       "      <th>source_link</th>\n",
       "      <th>hf_link</th>\n",
       "      <th>license</th>\n",
       "      <th>year</th>\n",
       "      <th>language</th>\n",
       "      <th>dialect</th>\n",
       "      <th>domain</th>\n",
       "      <th>...</th>\n",
       "      <th>dataloader</th>\n",
       "      <th>implemented</th>\n",
       "      <th>is_large</th>\n",
       "      <th>is_resource</th>\n",
       "      <th>is_default</th>\n",
       "      <th>is_broken</th>\n",
       "      <th>is_local</th>\n",
       "      <th>citation</th>\n",
       "      <th>homepage</th>\n",
       "      <th>metadata</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>CC100</td>\n",
       "      <td>False</td>\n",
       "      <td>https://data.statmt.org/cc-100/</td>\n",
       "      <td>https://huggingface.co/datasets/cc100</td>\n",
       "      <td>MIT</td>\n",
       "      <td>2020.0</td>\n",
       "      <td>ind, sun, jav</td>\n",
       "      <td>other</td>\n",
       "      <td>multi domain</td>\n",
       "      <td>...</td>\n",
       "      <td>cc100</td>\n",
       "      <td>1.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>@inproceedings{conneau-etal-2020-unsup...</td>\n",
       "      <td>https://data.statmt.org/cc-100/</td>\n",
       "      <td>MetaDict(data={'source': [NusantaraMetadata(sc...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>CC100</td>\n",
       "      <td>Indonesian</td>\n",
       "      <td>https://data.statmt.org/cc-100/</td>\n",
       "      <td>https://huggingface.co/datasets/cc100</td>\n",
       "      <td>Common Crawl's license</td>\n",
       "      <td>2020.0</td>\n",
       "      <td>ind</td>\n",
       "      <td>other</td>\n",
       "      <td>multi domain</td>\n",
       "      <td>...</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>4</td>\n",
       "      <td>CC100</td>\n",
       "      <td>Javanese</td>\n",
       "      <td>https://data.statmt.org/cc-100/</td>\n",
       "      <td>https://huggingface.co/datasets/cc100</td>\n",
       "      <td>Common Crawl's license</td>\n",
       "      <td>2020.0</td>\n",
       "      <td>jav</td>\n",
       "      <td>other</td>\n",
       "      <td>multi domain</td>\n",
       "      <td>...</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>4</td>\n",
       "      <td>CC100</td>\n",
       "      <td>Sundanese</td>\n",
       "      <td>https://data.statmt.org/cc-100/</td>\n",
       "      <td>https://huggingface.co/datasets/cc100</td>\n",
       "      <td>Common Crawl's license</td>\n",
       "      <td>2020.0</td>\n",
       "      <td>sun</td>\n",
       "      <td>other</td>\n",
       "      <td>multi domain</td>\n",
       "      <td>...</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>9</td>\n",
       "      <td>Customer Review (Natasha Skincare)</td>\n",
       "      <td>False</td>\n",
       "      <td>https://drive.google.com/file/d/1D1pHX7CxrI-eI...</td>\n",
       "      <td>False</td>\n",
       "      <td>Unknown</td>\n",
       "      <td>2017.0</td>\n",
       "      <td>ind</td>\n",
       "      <td>False</td>\n",
       "      <td>Social media</td>\n",
       "      <td>...</td>\n",
       "      <td>sentiment_nathasa_review</td>\n",
       "      <td>1.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>@article{nurlaila2018classification,\\n  title=...</td>\n",
       "      <td>https://jurnal.uns.ac.id/itsmart/article/viewF...</td>\n",
       "      <td>MetaDict(data={'source': [NusantaraMetadata(sc...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>153</th>\n",
       "      <td>116</td>\n",
       "      <td>UD_Indonesian-CSUI</td>\n",
       "      <td>False</td>\n",
       "      <td>https://github.com/UniversalDependencies/UD_In...</td>\n",
       "      <td>False</td>\n",
       "      <td>CC BY-SA 4.0</td>\n",
       "      <td>2020.0</td>\n",
       "      <td>ind</td>\n",
       "      <td>False</td>\n",
       "      <td>General, News articles</td>\n",
       "      <td>...</td>\n",
       "      <td>ud_id_csui</td>\n",
       "      <td>1.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>@article {10.3844/jcssp.2020.1585.1597,\\nautho...</td>\n",
       "      <td>https://github.com/UniversalDependencies/UD_In...</td>\n",
       "      <td>MetaDict(data={'source': [NusantaraMetadata(sc...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>158</th>\n",
       "      <td>121</td>\n",
       "      <td>WikiAnn</td>\n",
       "      <td>ind, jav, bug, min, bjn, sun, ace, tet, ms, ma...</td>\n",
       "      <td>https://drive.google.com/drive/folders/1Q-xdT9...</td>\n",
       "      <td>https://huggingface.co/datasets/wikiann</td>\n",
       "      <td>Apache-2.0 license</td>\n",
       "      <td>2017.0</td>\n",
       "      <td>ind, eng, sun, jav, min, bug, bjn, tpi, ace, t...</td>\n",
       "      <td>Banyumasan</td>\n",
       "      <td>Wiki articles</td>\n",
       "      <td>...</td>\n",
       "      <td>wikiann</td>\n",
       "      <td>1.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>@inproceedings{pan-etal-2017-cross,\\n    title...</td>\n",
       "      <td>https://github.com/afshinrahimi/mmner</td>\n",
       "      <td>MetaDict(data={'source': [NusantaraMetadata(sc...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>162</th>\n",
       "      <td>125</td>\n",
       "      <td>XCOPA</td>\n",
       "      <td>Indonesian</td>\n",
       "      <td>https://github.com/cambridgeltl/xcopa</td>\n",
       "      <td>https://huggingface.co/datasets/xcopa</td>\n",
       "      <td>Unknown</td>\n",
       "      <td>2021.0</td>\n",
       "      <td>ind</td>\n",
       "      <td>False</td>\n",
       "      <td>General</td>\n",
       "      <td>...</td>\n",
       "      <td>xcopa</td>\n",
       "      <td>1.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>@inproceedings{ponti2020xcopa,\\n  title={{XCOP...</td>\n",
       "      <td>https://github.com/cambridgeltl/xcopa</td>\n",
       "      <td>MetaDict(data={'source': [NusantaraMetadata(sc...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>163</th>\n",
       "      <td>126</td>\n",
       "      <td>XL-Sum</td>\n",
       "      <td>Indonesian</td>\n",
       "      <td>https://github.com/csebuetnlp/xl-sum</td>\n",
       "      <td>https://huggingface.co/datasets/csebuetnlp/xlsum</td>\n",
       "      <td>CC-BY-NC-SA 4.0</td>\n",
       "      <td>2021.0</td>\n",
       "      <td>ind</td>\n",
       "      <td>False</td>\n",
       "      <td>News articles</td>\n",
       "      <td>...</td>\n",
       "      <td>xl_sum</td>\n",
       "      <td>1.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>@inproceedings{hasan2021xl,\\n  title={XL-Sum: ...</td>\n",
       "      <td>https://github.com/csebuetnlp/xl-sum</td>\n",
       "      <td>MetaDict(data={'source': [NusantaraMetadata(sc...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>164</th>\n",
       "      <td>127</td>\n",
       "      <td>XPersona Id</td>\n",
       "      <td>False</td>\n",
       "      <td>https://storage.googleapis.com/babert-pretrain...</td>\n",
       "      <td>-</td>\n",
       "      <td>CC-BY-SA 4.0</td>\n",
       "      <td>2021.0</td>\n",
       "      <td>ind</td>\n",
       "      <td>colloquial</td>\n",
       "      <td>conversational</td>\n",
       "      <td>...</td>\n",
       "      <td>xpersona_id</td>\n",
       "      <td>1.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>@article{lin2020xpersona,\\n  title={XPersona: ...</td>\n",
       "      <td></td>\n",
       "      <td>MetaDict(data={'source': [NusantaraMetadata(sc...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>98 rows × 34 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      id                                name  \\\n",
       "3      4                               CC100   \n",
       "4      4                               CC100   \n",
       "5      4                               CC100   \n",
       "6      4                               CC100   \n",
       "11     9  Customer Review (Natasha Skincare)   \n",
       "..   ...                                 ...   \n",
       "153  116                  UD_Indonesian-CSUI   \n",
       "158  121                             WikiAnn   \n",
       "162  125                               XCOPA   \n",
       "163  126                              XL-Sum   \n",
       "164  127                         XPersona Id   \n",
       "\n",
       "                                               subsets  \\\n",
       "3                                                False   \n",
       "4                                           Indonesian   \n",
       "5                                             Javanese   \n",
       "6                                            Sundanese   \n",
       "11                                               False   \n",
       "..                                                 ...   \n",
       "153                                              False   \n",
       "158  ind, jav, bug, min, bjn, sun, ace, tet, ms, ma...   \n",
       "162                                         Indonesian   \n",
       "163                                         Indonesian   \n",
       "164                                              False   \n",
       "\n",
       "                                           source_link  \\\n",
       "3                      https://data.statmt.org/cc-100/   \n",
       "4                      https://data.statmt.org/cc-100/   \n",
       "5                      https://data.statmt.org/cc-100/   \n",
       "6                      https://data.statmt.org/cc-100/   \n",
       "11   https://drive.google.com/file/d/1D1pHX7CxrI-eI...   \n",
       "..                                                 ...   \n",
       "153  https://github.com/UniversalDependencies/UD_In...   \n",
       "158  https://drive.google.com/drive/folders/1Q-xdT9...   \n",
       "162              https://github.com/cambridgeltl/xcopa   \n",
       "163               https://github.com/csebuetnlp/xl-sum   \n",
       "164  https://storage.googleapis.com/babert-pretrain...   \n",
       "\n",
       "                                              hf_link                 license  \\\n",
       "3               https://huggingface.co/datasets/cc100                     MIT   \n",
       "4               https://huggingface.co/datasets/cc100  Common Crawl's license   \n",
       "5               https://huggingface.co/datasets/cc100  Common Crawl's license   \n",
       "6               https://huggingface.co/datasets/cc100  Common Crawl's license   \n",
       "11                                              False                 Unknown   \n",
       "..                                                ...                     ...   \n",
       "153                                             False            CC BY-SA 4.0   \n",
       "158           https://huggingface.co/datasets/wikiann      Apache-2.0 license   \n",
       "162             https://huggingface.co/datasets/xcopa                 Unknown   \n",
       "163  https://huggingface.co/datasets/csebuetnlp/xlsum         CC-BY-NC-SA 4.0   \n",
       "164                                                 -            CC-BY-SA 4.0   \n",
       "\n",
       "       year                                           language     dialect  \\\n",
       "3    2020.0                                      ind, sun, jav       other   \n",
       "4    2020.0                                                ind       other   \n",
       "5    2020.0                                                jav       other   \n",
       "6    2020.0                                                sun       other   \n",
       "11   2017.0                                                ind       False   \n",
       "..      ...                                                ...         ...   \n",
       "153  2020.0                                                ind       False   \n",
       "158  2017.0  ind, eng, sun, jav, min, bug, bjn, tpi, ace, t...  Banyumasan   \n",
       "162  2021.0                                                ind       False   \n",
       "163  2021.0                                                ind       False   \n",
       "164  2021.0                                                ind  colloquial   \n",
       "\n",
       "                     domain  ...                dataloader implemented  \\\n",
       "3              multi domain  ...                     cc100         1.0   \n",
       "4              multi domain  ...                     False       False   \n",
       "5              multi domain  ...                     False       False   \n",
       "6              multi domain  ...                     False       False   \n",
       "11             Social media  ...  sentiment_nathasa_review         1.0   \n",
       "..                      ...  ...                       ...         ...   \n",
       "153  General, News articles  ...                ud_id_csui         1.0   \n",
       "158           Wiki articles  ...                   wikiann         1.0   \n",
       "162                 General  ...                     xcopa         1.0   \n",
       "163           News articles  ...                    xl_sum         1.0   \n",
       "164          conversational  ...               xpersona_id         1.0   \n",
       "\n",
       "    is_large is_resource is_default is_broken is_local  \\\n",
       "3      False       False      False     False    False   \n",
       "4      False       False      False     False    False   \n",
       "5      False       False      False     False    False   \n",
       "6      False       False      False     False    False   \n",
       "11     False       False      False     False    False   \n",
       "..       ...         ...        ...       ...      ...   \n",
       "153    False       False      False     False    False   \n",
       "158    False       False      False     False    False   \n",
       "162    False       False      False     False    False   \n",
       "163    False       False      False     False    False   \n",
       "164    False       False      False     False    False   \n",
       "\n",
       "                                              citation  \\\n",
       "3            @inproceedings{conneau-etal-2020-unsup...   \n",
       "4                                                False   \n",
       "5                                                False   \n",
       "6                                                False   \n",
       "11   @article{nurlaila2018classification,\\n  title=...   \n",
       "..                                                 ...   \n",
       "153  @article {10.3844/jcssp.2020.1585.1597,\\nautho...   \n",
       "158  @inproceedings{pan-etal-2017-cross,\\n    title...   \n",
       "162  @inproceedings{ponti2020xcopa,\\n  title={{XCOP...   \n",
       "163  @inproceedings{hasan2021xl,\\n  title={XL-Sum: ...   \n",
       "164  @article{lin2020xpersona,\\n  title={XPersona: ...   \n",
       "\n",
       "                                              homepage  \\\n",
       "3                      https://data.statmt.org/cc-100/   \n",
       "4                                                False   \n",
       "5                                                False   \n",
       "6                                                False   \n",
       "11   https://jurnal.uns.ac.id/itsmart/article/viewF...   \n",
       "..                                                 ...   \n",
       "153  https://github.com/UniversalDependencies/UD_In...   \n",
       "158              https://github.com/afshinrahimi/mmner   \n",
       "162              https://github.com/cambridgeltl/xcopa   \n",
       "163               https://github.com/csebuetnlp/xl-sum   \n",
       "164                                                      \n",
       "\n",
       "                                              metadata  \n",
       "3    MetaDict(data={'source': [NusantaraMetadata(sc...  \n",
       "4                                                  NaN  \n",
       "5                                                  NaN  \n",
       "6                                                  NaN  \n",
       "11   MetaDict(data={'source': [NusantaraMetadata(sc...  \n",
       "..                                                 ...  \n",
       "153  MetaDict(data={'source': [NusantaraMetadata(sc...  \n",
       "158  MetaDict(data={'source': [NusantaraMetadata(sc...  \n",
       "162  MetaDict(data={'source': [NusantaraMetadata(sc...  \n",
       "163  MetaDict(data={'source': [NusantaraMetadata(sc...  \n",
       "164  MetaDict(data={'source': [NusantaraMetadata(sc...  \n",
       "\n",
       "[98 rows x 34 columns]"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Filter & load all Indonesian sentiment analysis task\n",
    "lang = 'ind'\n",
    "task = Tasks.SENTIMENT_ANALYSIS\n",
    "\n",
    "filtered_df = meta_df.loc[\n",
    "    meta_df.is_splitted\n",
    "]\n",
    "\n",
    "schema = f'nusantara_{TASK_TO_SCHEMA[task].lower()}'\n",
    "datasets = {}\n",
    "for metas in filtered_df.metadata:\n",
    "    if schema in metas.data:\n",
    "        for meta in metas.data[schema]:\n",
    "            if len(meta.languages) > 1:\n",
    "                if lang in meta.config.name:\n",
    "                    datasets[meta.config.name] = meta.load_dataset()\n",
    "            else:\n",
    "                datasets[meta.config.name] = meta.load_dataset()\n",
    "datasets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "663f21ff-a18b-4448-ab25-5764939c8826",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>name</th>\n",
       "      <th>subsets</th>\n",
       "      <th>source_link</th>\n",
       "      <th>hf_link</th>\n",
       "      <th>license</th>\n",
       "      <th>year</th>\n",
       "      <th>language</th>\n",
       "      <th>dialect</th>\n",
       "      <th>domain</th>\n",
       "      <th>...</th>\n",
       "      <th>dataloader</th>\n",
       "      <th>implemented</th>\n",
       "      <th>is_large</th>\n",
       "      <th>is_resource</th>\n",
       "      <th>is_default</th>\n",
       "      <th>is_broken</th>\n",
       "      <th>is_local</th>\n",
       "      <th>citation</th>\n",
       "      <th>homepage</th>\n",
       "      <th>metadata</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>CC100</td>\n",
       "      <td>False</td>\n",
       "      <td>https://data.statmt.org/cc-100/</td>\n",
       "      <td>https://huggingface.co/datasets/cc100</td>\n",
       "      <td>MIT</td>\n",
       "      <td>2020.0</td>\n",
       "      <td>ind, sun, jav</td>\n",
       "      <td>other</td>\n",
       "      <td>multi domain</td>\n",
       "      <td>...</td>\n",
       "      <td>cc100</td>\n",
       "      <td>1.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>@inproceedings{conneau-etal-2020-unsup...</td>\n",
       "      <td>https://data.statmt.org/cc-100/</td>\n",
       "      <td>MetaDict(data={'source': [NusantaraMetadata(sc...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>CC100</td>\n",
       "      <td>Indonesian</td>\n",
       "      <td>https://data.statmt.org/cc-100/</td>\n",
       "      <td>https://huggingface.co/datasets/cc100</td>\n",
       "      <td>Common Crawl's license</td>\n",
       "      <td>2020.0</td>\n",
       "      <td>ind</td>\n",
       "      <td>other</td>\n",
       "      <td>multi domain</td>\n",
       "      <td>...</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>4</td>\n",
       "      <td>CC100</td>\n",
       "      <td>Javanese</td>\n",
       "      <td>https://data.statmt.org/cc-100/</td>\n",
       "      <td>https://huggingface.co/datasets/cc100</td>\n",
       "      <td>Common Crawl's license</td>\n",
       "      <td>2020.0</td>\n",
       "      <td>jav</td>\n",
       "      <td>other</td>\n",
       "      <td>multi domain</td>\n",
       "      <td>...</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>4</td>\n",
       "      <td>CC100</td>\n",
       "      <td>Sundanese</td>\n",
       "      <td>https://data.statmt.org/cc-100/</td>\n",
       "      <td>https://huggingface.co/datasets/cc100</td>\n",
       "      <td>Common Crawl's license</td>\n",
       "      <td>2020.0</td>\n",
       "      <td>sun</td>\n",
       "      <td>other</td>\n",
       "      <td>multi domain</td>\n",
       "      <td>...</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>9</td>\n",
       "      <td>Customer Review (Natasha Skincare)</td>\n",
       "      <td>False</td>\n",
       "      <td>https://drive.google.com/file/d/1D1pHX7CxrI-eI...</td>\n",
       "      <td>False</td>\n",
       "      <td>Unknown</td>\n",
       "      <td>2017.0</td>\n",
       "      <td>ind</td>\n",
       "      <td>False</td>\n",
       "      <td>Social media</td>\n",
       "      <td>...</td>\n",
       "      <td>sentiment_nathasa_review</td>\n",
       "      <td>1.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>@article{nurlaila2018classification,\\n  title=...</td>\n",
       "      <td>https://jurnal.uns.ac.id/itsmart/article/viewF...</td>\n",
       "      <td>MetaDict(data={'source': [NusantaraMetadata(sc...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>153</th>\n",
       "      <td>116</td>\n",
       "      <td>UD_Indonesian-CSUI</td>\n",
       "      <td>False</td>\n",
       "      <td>https://github.com/UniversalDependencies/UD_In...</td>\n",
       "      <td>False</td>\n",
       "      <td>CC BY-SA 4.0</td>\n",
       "      <td>2020.0</td>\n",
       "      <td>ind</td>\n",
       "      <td>False</td>\n",
       "      <td>General, News articles</td>\n",
       "      <td>...</td>\n",
       "      <td>ud_id_csui</td>\n",
       "      <td>1.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>@article {10.3844/jcssp.2020.1585.1597,\\nautho...</td>\n",
       "      <td>https://github.com/UniversalDependencies/UD_In...</td>\n",
       "      <td>MetaDict(data={'source': [NusantaraMetadata(sc...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>158</th>\n",
       "      <td>121</td>\n",
       "      <td>WikiAnn</td>\n",
       "      <td>ind, jav, bug, min, bjn, sun, ace, tet, ms, ma...</td>\n",
       "      <td>https://drive.google.com/drive/folders/1Q-xdT9...</td>\n",
       "      <td>https://huggingface.co/datasets/wikiann</td>\n",
       "      <td>Apache-2.0 license</td>\n",
       "      <td>2017.0</td>\n",
       "      <td>ind, eng, sun, jav, min, bug, bjn, tpi, ace, t...</td>\n",
       "      <td>Banyumasan</td>\n",
       "      <td>Wiki articles</td>\n",
       "      <td>...</td>\n",
       "      <td>wikiann</td>\n",
       "      <td>1.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>@inproceedings{pan-etal-2017-cross,\\n    title...</td>\n",
       "      <td>https://github.com/afshinrahimi/mmner</td>\n",
       "      <td>MetaDict(data={'source': [NusantaraMetadata(sc...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>162</th>\n",
       "      <td>125</td>\n",
       "      <td>XCOPA</td>\n",
       "      <td>Indonesian</td>\n",
       "      <td>https://github.com/cambridgeltl/xcopa</td>\n",
       "      <td>https://huggingface.co/datasets/xcopa</td>\n",
       "      <td>Unknown</td>\n",
       "      <td>2021.0</td>\n",
       "      <td>ind</td>\n",
       "      <td>False</td>\n",
       "      <td>General</td>\n",
       "      <td>...</td>\n",
       "      <td>xcopa</td>\n",
       "      <td>1.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>@inproceedings{ponti2020xcopa,\\n  title={{XCOP...</td>\n",
       "      <td>https://github.com/cambridgeltl/xcopa</td>\n",
       "      <td>MetaDict(data={'source': [NusantaraMetadata(sc...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>163</th>\n",
       "      <td>126</td>\n",
       "      <td>XL-Sum</td>\n",
       "      <td>Indonesian</td>\n",
       "      <td>https://github.com/csebuetnlp/xl-sum</td>\n",
       "      <td>https://huggingface.co/datasets/csebuetnlp/xlsum</td>\n",
       "      <td>CC-BY-NC-SA 4.0</td>\n",
       "      <td>2021.0</td>\n",
       "      <td>ind</td>\n",
       "      <td>False</td>\n",
       "      <td>News articles</td>\n",
       "      <td>...</td>\n",
       "      <td>xl_sum</td>\n",
       "      <td>1.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>@inproceedings{hasan2021xl,\\n  title={XL-Sum: ...</td>\n",
       "      <td>https://github.com/csebuetnlp/xl-sum</td>\n",
       "      <td>MetaDict(data={'source': [NusantaraMetadata(sc...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>164</th>\n",
       "      <td>127</td>\n",
       "      <td>XPersona Id</td>\n",
       "      <td>False</td>\n",
       "      <td>https://storage.googleapis.com/babert-pretrain...</td>\n",
       "      <td>-</td>\n",
       "      <td>CC-BY-SA 4.0</td>\n",
       "      <td>2021.0</td>\n",
       "      <td>ind</td>\n",
       "      <td>colloquial</td>\n",
       "      <td>conversational</td>\n",
       "      <td>...</td>\n",
       "      <td>xpersona_id</td>\n",
       "      <td>1.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>@article{lin2020xpersona,\\n  title={XPersona: ...</td>\n",
       "      <td></td>\n",
       "      <td>MetaDict(data={'source': [NusantaraMetadata(sc...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>98 rows × 34 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      id                                name  \\\n",
       "3      4                               CC100   \n",
       "4      4                               CC100   \n",
       "5      4                               CC100   \n",
       "6      4                               CC100   \n",
       "11     9  Customer Review (Natasha Skincare)   \n",
       "..   ...                                 ...   \n",
       "153  116                  UD_Indonesian-CSUI   \n",
       "158  121                             WikiAnn   \n",
       "162  125                               XCOPA   \n",
       "163  126                              XL-Sum   \n",
       "164  127                         XPersona Id   \n",
       "\n",
       "                                               subsets  \\\n",
       "3                                                False   \n",
       "4                                           Indonesian   \n",
       "5                                             Javanese   \n",
       "6                                            Sundanese   \n",
       "11                                               False   \n",
       "..                                                 ...   \n",
       "153                                              False   \n",
       "158  ind, jav, bug, min, bjn, sun, ace, tet, ms, ma...   \n",
       "162                                         Indonesian   \n",
       "163                                         Indonesian   \n",
       "164                                              False   \n",
       "\n",
       "                                           source_link  \\\n",
       "3                      https://data.statmt.org/cc-100/   \n",
       "4                      https://data.statmt.org/cc-100/   \n",
       "5                      https://data.statmt.org/cc-100/   \n",
       "6                      https://data.statmt.org/cc-100/   \n",
       "11   https://drive.google.com/file/d/1D1pHX7CxrI-eI...   \n",
       "..                                                 ...   \n",
       "153  https://github.com/UniversalDependencies/UD_In...   \n",
       "158  https://drive.google.com/drive/folders/1Q-xdT9...   \n",
       "162              https://github.com/cambridgeltl/xcopa   \n",
       "163               https://github.com/csebuetnlp/xl-sum   \n",
       "164  https://storage.googleapis.com/babert-pretrain...   \n",
       "\n",
       "                                              hf_link                 license  \\\n",
       "3               https://huggingface.co/datasets/cc100                     MIT   \n",
       "4               https://huggingface.co/datasets/cc100  Common Crawl's license   \n",
       "5               https://huggingface.co/datasets/cc100  Common Crawl's license   \n",
       "6               https://huggingface.co/datasets/cc100  Common Crawl's license   \n",
       "11                                              False                 Unknown   \n",
       "..                                                ...                     ...   \n",
       "153                                             False            CC BY-SA 4.0   \n",
       "158           https://huggingface.co/datasets/wikiann      Apache-2.0 license   \n",
       "162             https://huggingface.co/datasets/xcopa                 Unknown   \n",
       "163  https://huggingface.co/datasets/csebuetnlp/xlsum         CC-BY-NC-SA 4.0   \n",
       "164                                                 -            CC-BY-SA 4.0   \n",
       "\n",
       "       year                                           language     dialect  \\\n",
       "3    2020.0                                      ind, sun, jav       other   \n",
       "4    2020.0                                                ind       other   \n",
       "5    2020.0                                                jav       other   \n",
       "6    2020.0                                                sun       other   \n",
       "11   2017.0                                                ind       False   \n",
       "..      ...                                                ...         ...   \n",
       "153  2020.0                                                ind       False   \n",
       "158  2017.0  ind, eng, sun, jav, min, bug, bjn, tpi, ace, t...  Banyumasan   \n",
       "162  2021.0                                                ind       False   \n",
       "163  2021.0                                                ind       False   \n",
       "164  2021.0                                                ind  colloquial   \n",
       "\n",
       "                     domain  ...                dataloader implemented  \\\n",
       "3              multi domain  ...                     cc100         1.0   \n",
       "4              multi domain  ...                     False       False   \n",
       "5              multi domain  ...                     False       False   \n",
       "6              multi domain  ...                     False       False   \n",
       "11             Social media  ...  sentiment_nathasa_review         1.0   \n",
       "..                      ...  ...                       ...         ...   \n",
       "153  General, News articles  ...                ud_id_csui         1.0   \n",
       "158           Wiki articles  ...                   wikiann         1.0   \n",
       "162                 General  ...                     xcopa         1.0   \n",
       "163           News articles  ...                    xl_sum         1.0   \n",
       "164          conversational  ...               xpersona_id         1.0   \n",
       "\n",
       "    is_large is_resource is_default is_broken is_local  \\\n",
       "3      False       False      False     False    False   \n",
       "4      False       False      False     False    False   \n",
       "5      False       False      False     False    False   \n",
       "6      False       False      False     False    False   \n",
       "11     False       False      False     False    False   \n",
       "..       ...         ...        ...       ...      ...   \n",
       "153    False       False      False     False    False   \n",
       "158    False       False      False     False    False   \n",
       "162    False       False      False     False    False   \n",
       "163    False       False      False     False    False   \n",
       "164    False       False      False     False    False   \n",
       "\n",
       "                                              citation  \\\n",
       "3            @inproceedings{conneau-etal-2020-unsup...   \n",
       "4                                                False   \n",
       "5                                                False   \n",
       "6                                                False   \n",
       "11   @article{nurlaila2018classification,\\n  title=...   \n",
       "..                                                 ...   \n",
       "153  @article {10.3844/jcssp.2020.1585.1597,\\nautho...   \n",
       "158  @inproceedings{pan-etal-2017-cross,\\n    title...   \n",
       "162  @inproceedings{ponti2020xcopa,\\n  title={{XCOP...   \n",
       "163  @inproceedings{hasan2021xl,\\n  title={XL-Sum: ...   \n",
       "164  @article{lin2020xpersona,\\n  title={XPersona: ...   \n",
       "\n",
       "                                              homepage  \\\n",
       "3                      https://data.statmt.org/cc-100/   \n",
       "4                                                False   \n",
       "5                                                False   \n",
       "6                                                False   \n",
       "11   https://jurnal.uns.ac.id/itsmart/article/viewF...   \n",
       "..                                                 ...   \n",
       "153  https://github.com/UniversalDependencies/UD_In...   \n",
       "158              https://github.com/afshinrahimi/mmner   \n",
       "162              https://github.com/cambridgeltl/xcopa   \n",
       "163               https://github.com/csebuetnlp/xl-sum   \n",
       "164                                                      \n",
       "\n",
       "                                              metadata  \n",
       "3    MetaDict(data={'source': [NusantaraMetadata(sc...  \n",
       "4                                                  NaN  \n",
       "5                                                  NaN  \n",
       "6                                                  NaN  \n",
       "11   MetaDict(data={'source': [NusantaraMetadata(sc...  \n",
       "..                                                 ...  \n",
       "153  MetaDict(data={'source': [NusantaraMetadata(sc...  \n",
       "158  MetaDict(data={'source': [NusantaraMetadata(sc...  \n",
       "162  MetaDict(data={'source': [NusantaraMetadata(sc...  \n",
       "163  MetaDict(data={'source': [NusantaraMetadata(sc...  \n",
       "164  MetaDict(data={'source': [NusantaraMetadata(sc...  \n",
       "\n",
       "[98 rows x 34 columns]"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "meta_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "cbcfaff7-0688-484c-b5af-f35636b0906c",
   "metadata": {},
   "outputs": [],
   "source": [
    "TEXT_CLASSIFICATION_TASKS = [\n",
    "    # Single Label Classification\n",
    "    'emot_nusantara_text',\n",
    "    'emotcmt_nusantara_text',\n",
    "    'emotion_id_opinion_nusantara_text',\n",
    "    'id_abusive_nusantara_text',\n",
    "    'id_clickbait_nusantara_text',\n",
    "    'id_google_play_review_nusantara_text',\n",
    "    'id_google_play_review_posneg_nusantara_text',\n",
    "    'id_hatespeech_nusantara_text',\n",
    "    'imdb_jv_nusantara_text',\n",
    "    'indolem_sentiment_nusantara_text',\n",
    "    'jadi_ide_nusantara_text',\n",
    "    'nusax_senti_ace_nusantara_text',\n",
    "    'nusax_senti_ban_nusantara_text',\n",
    "    'nusax_senti_bjn_nusantara_text',\n",
    "    'nusax_senti_bug_nusantara_text',\n",
    "    'nusax_senti_eng_nusantara_text',\n",
    "    'nusax_senti_ind_nusantara_text',\n",
    "    'nusax_senti_jav_nusantara_text',\n",
    "    'nusax_senti_mad_nusantara_text',\n",
    "    'nusax_senti_min_nusantara_text',\n",
    "    'nusax_senti_nij_nusantara_text',\n",
    "    'nusax_senti_sun_nusantara_text',\n",
    "    'nusax_senti_bbc_nusantara_text',\n",
    "    'nusax_senti_nusantara_text',\n",
    "    'sentiment_nathasa_review_nusantara_text',\n",
    "    'smsa_nusantara_text',\n",
    "    # Pair Sentence Classification\n",
    "    'id_stance_nusantara_pairs',\n",
    "    'indolem_ntp_nusantara_pairs',\n",
    "    'indonli_nusantara_pairs',\n",
    "    'id_sts_nusantara_pairs_score',\n",
    "    # Multilabel Classification\n",
    "    'casa_nusantara_text_multi',\n",
    "    'hoasa_nusantara_text_multi',\n",
    "    'netifier_nusantara_text_multi',\n",
    "    'id_multilabel_hs_nusantara_text_multi',\n",
    "]\n",
    "\n",
    "TEXT_GENERATION_TASKS = [\n",
    "    # MT\n",
    "    'bible_en_id_nusantara_t2t',\n",
    "    'bible_jv_id_nusantara_t2t',\n",
    "    'bible_su_id_nusantara_t2t',\n",
    "    'id_panl_bppt_nusantara_t2t',\n",
    "    'indo_general_mt_en_id_nusantara_t2t',\n",
    "    'indo_religious_mt_en_id_nusantara_t2t',\n",
    "    'minangnlp_mt_nusantara_t2t',\n",
    "    'news_en_id_nusantara_t2t',\n",
    "    'nusax_mt_ace_ind_nusantara_t2t',\n",
    "    'nusax_mt_ban_ind_nusantara_t2t',\n",
    "    'nusax_mt_bjn_ind_nusantara_t2t',\n",
    "    'nusax_mt_bug_ind_nusantara_t2t',\n",
    "    'nusax_mt_eng_ind_nusantara_t2t',\n",
    "    'nusax_mt_ind_ace_nusantara_t2t',\n",
    "    'nusax_mt_ind_ban_nusantara_t2t',\n",
    "    'nusax_mt_ind_bjn_nusantara_t2t',\n",
    "    'nusax_mt_ind_bug_nusantara_t2t',\n",
    "    'nusax_mt_ind_eng_nusantara_t2t',\n",
    "    'nusax_mt_ind_jav_nusantara_t2t',\n",
    "    'nusax_mt_ind_mad_nusantara_t2t',\n",
    "    'nusax_mt_ind_min_nusantara_t2t',\n",
    "    'nusax_mt_ind_nij_nusantara_t2t',\n",
    "    'nusax_mt_ind_sun_nusantara_t2t',\n",
    "    'nusax_mt_ind_bbc_nusantara_t2t',\n",
    "    'nusax_mt_jav_ind_nusantara_t2t',\n",
    "    'nusax_mt_mad_ind_nusantara_t2t',\n",
    "    'nusax_mt_min_ind_nusantara_t2t',\n",
    "    'nusax_mt_nij_ind_nusantara_t2t',\n",
    "    'nusax_mt_sun_ind_nusantara_t2t',\n",
    "    'nusax_mt_bbc_ind_nusantara_t2t',\n",
    "    'parallel_su_id_nusantara_t2t',\n",
    "    'ted_en_id_nusantara_t2t',\n",
    "    'ud_id_csui_nusantara_t2t',\n",
    "    # Paraphrasing\n",
    "    # 'id_qqp_nusantara_t2t',\n",
    "    # 'paracotta_id_nusantara_t2t',\n",
    "    'stif_indonesia_nusantara_t2t',\n",
    "    # Summarization\n",
    "    'indosum_fold0_nusantara_t2t',\n",
    "    'xl_sum_nusantara_t2t',\n",
    "    # Dialogue System\n",
    "    'xpersona_id_nusantara_t2t',\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "4b474b49-db85-4e43-b44a-97b5204b5643",
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'conhelps' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "Input \u001b[0;32mIn [3]\u001b[0m, in \u001b[0;36m<cell line: 1>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      1\u001b[0m text_classification_datasets \u001b[38;5;241m=\u001b[39m {\n\u001b[0;32m----> 2\u001b[0m     helper\u001b[38;5;241m.\u001b[39mconfig\u001b[38;5;241m.\u001b[39mname: helper\u001b[38;5;241m.\u001b[39mload_dataset() \u001b[38;5;28;01mfor\u001b[39;00m helper \u001b[38;5;129;01min\u001b[39;00m \u001b[43mconhelps\u001b[49m\u001b[38;5;241m.\u001b[39mfiltered(\u001b[38;5;28;01mlambda\u001b[39;00m x: x\u001b[38;5;241m.\u001b[39mconfig\u001b[38;5;241m.\u001b[39mname \u001b[38;5;129;01min\u001b[39;00m TEXT_CLASSIFICATION_TASKS)\n\u001b[1;32m      3\u001b[0m }\n",
      "\u001b[0;31mNameError\u001b[0m: name 'conhelps' is not defined"
     ]
    }
   ],
   "source": [
    "text_classification_datasets = {\n",
    "    helper.config.name: helper.load_dataset() for helper in conhelps.filtered(lambda x: x.config.name in TEXT_CLASSIFICATION_TASKS)\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "e0882dbf-34ac-4023-9d4b-685cdec44772",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Downloading and preparing dataset bible_en_id/bible_en_id_nusantara_t2t to /home/samuel/.cache/huggingface/datasets/bible_en_id/bible_en_id_nusantara_t2t/1.0.0/0914cfc955aeb37cf55222ac526adcdeea078a3d4fac89b7e24be78197f28584...\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Generating train split: 0 examples [00:00, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Generating validation split: 0 examples [00:00, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Generating test split: 0 examples [00:00, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Dataset bible_en_id downloaded and prepared to /home/samuel/.cache/huggingface/datasets/bible_en_id/bible_en_id_nusantara_t2t/1.0.0/0914cfc955aeb37cf55222ac526adcdeea078a3d4fac89b7e24be78197f28584. Subsequent calls will reuse this data.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "db973464135549458604b3d3ce6e7a4e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/3 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Downloading and preparing dataset bible_en_id/bible_jv_id_nusantara_t2t to /home/samuel/.cache/huggingface/datasets/bible_en_id/bible_jv_id_nusantara_t2t/1.0.0/94cb64872abc2af5dbc51e125560e3584ad3490a12276f6274811ffc690b6577...\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Generating train split: 0 examples [00:00, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Generating validation split: 0 examples [00:00, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Generating test split: 0 examples [00:00, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Dataset bible_en_id downloaded and prepared to /home/samuel/.cache/huggingface/datasets/bible_en_id/bible_jv_id_nusantara_t2t/1.0.0/94cb64872abc2af5dbc51e125560e3584ad3490a12276f6274811ffc690b6577. Subsequent calls will reuse this data.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a83f81947d144346b6c6c23f8cdcc28a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/3 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Downloading and preparing dataset bible_su_id/bible_su_id_nusantara_t2t to /home/samuel/.cache/huggingface/datasets/bible_su_id/bible_su_id_nusantara_t2t/1.0.0/afe00cbc1bbbc8c3fdcbfa5d75ef6d8eca68cbb42bfdfbefb1efe66777190b55...\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Generating train split: 0 examples [00:00, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Generating validation split: 0 examples [00:00, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Generating test split: 0 examples [00:00, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Dataset bible_su_id downloaded and prepared to /home/samuel/.cache/huggingface/datasets/bible_su_id/bible_su_id_nusantara_t2t/1.0.0/afe00cbc1bbbc8c3fdcbfa5d75ef6d8eca68cbb42bfdfbefb1efe66777190b55. Subsequent calls will reuse this data.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d448372a7de24622ab81feca5b93985b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/3 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Downloading and preparing dataset id_panl_bppt/id_panl_bppt_nusantara_t2t to /home/samuel/.cache/huggingface/datasets/id_panl_bppt/id_panl_bppt_nusantara_t2t/1.0.0/3e0338dbc7d5c3b44c5ca68d5374c363476db1453d10206c1c04f1bbc4b57d0b...\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "328d45f130a74580bcb3ff4ced833057",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading data:   0%|          | 0.00/2.37M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Generating train split: 0 examples [00:00, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Dataset id_panl_bppt downloaded and prepared to /home/samuel/.cache/huggingface/datasets/id_panl_bppt/id_panl_bppt_nusantara_t2t/1.0.0/3e0338dbc7d5c3b44c5ca68d5374c363476db1453d10206c1c04f1bbc4b57d0b. Subsequent calls will reuse this data.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5a59e93113e04000a24e13145c919635",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/1 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Downloading and preparing dataset id_quora_question_pairs/id_qqp_nusantara_t2t to /home/samuel/.cache/huggingface/datasets/id_quora_question_pairs/id_qqp_nusantara_t2t/1.0.0/eed59379992ba1ede73b9720cadb64df39bbe8f05c831517b241e7920135147d...\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ec168bf731634fe6916744420661c9f2",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading data files:   0%|          | 0/2 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f59b6f2ef32943d3a0a6a445b58e6b4c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Extracting data files:   0%|          | 0/2 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Generating train split: 0 examples [00:00, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Generating validation split: 0 examples [00:00, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Dataset id_quora_question_pairs downloaded and prepared to /home/samuel/.cache/huggingface/datasets/id_quora_question_pairs/id_qqp_nusantara_t2t/1.0.0/eed59379992ba1ede73b9720cadb64df39bbe8f05c831517b241e7920135147d. Subsequent calls will reuse this data.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f30c3a25e6cc4284b282b2c839dc9868",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/2 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:datasets.builder:Reusing dataset indo_general_mt_en_id (/home/samuel/.cache/huggingface/datasets/indo_general_mt_en_id/indo_general_mt_en_id_nusantara_t2t/1.0.0/8997ff3f81a5107662a8a4d0e405a1e053a076327735146c76b56dd6c263e4c1)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8cbd3e4a4d324fcea4f39a64811f322c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/3 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Downloading and preparing dataset indo_religious_mt_en_id/indo_religious_mt_en_id_nusantara_t2t to /home/samuel/.cache/huggingface/datasets/indo_religious_mt_en_id/indo_religious_mt_en_id_nusantara_t2t/1.0.0/3436a2763dcd62dd8a14e3c8d12e1b99fad9e4d82b32d68131c87b5b77b581f7...\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7b76578a045b4d60a57e9a4f8d61b68e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading data:   0%|          | 0.00/164k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "bcec99f0268c4e7f837b57ffec37be14",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading data:   0%|          | 0.00/177k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "db4e8c995c2b4e98ac7abeaa33ec796e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading data:   0%|          | 0.00/169k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "402a70ed1dc94416993288b00d1e0695",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading data:   0%|          | 0.00/186k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "095fff00d0934d4bbc7f7593b6e8b653",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading data:   0%|          | 0.00/17.2M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4a59023c7c12416aa11852a22520c39f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading data:   0%|          | 0.00/2.85M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "221242f50e9c4d38b48d881feab5a7d0",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading data:   0%|          | 0.00/16.9M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4fd69a3449544ed3ad4f5d1b8cfa5626",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading data:   0%|          | 0.00/3.42M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Generating test split: 0 examples [00:00, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Generating validation split: 0 examples [00:00, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Generating train split: 0 examples [00:00, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Dataset indo_religious_mt_en_id downloaded and prepared to /home/samuel/.cache/huggingface/datasets/indo_religious_mt_en_id/indo_religious_mt_en_id_nusantara_t2t/1.0.0/3436a2763dcd62dd8a14e3c8d12e1b99fad9e4d82b32d68131c87b5b77b581f7. Subsequent calls will reuse this data.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0ce79a4a6eb24dc292920c69aa22f410",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/3 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Downloading and preparing dataset indo_sum/indosum_fold0_nusantara_t2t to /home/samuel/.cache/huggingface/datasets/indo_sum/indosum_fold0_nusantara_t2t/1.0.0/f779ca1f657fd0562f1534b73b03183ed41ad8f80f4b8003b0a19bd0e0888633...\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "fa22e23877d2466db5f51c9c9a7b4621",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading data:   0%|          | 0.00/96.0M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Generating train split: 0 examples [00:00, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Generating test split: 0 examples [00:00, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "48bdfa1245054979a74ec92becc481bb",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Generating validation split: 0 examples [00:00, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Dataset indo_sum downloaded and prepared to /home/samuel/.cache/huggingface/datasets/indo_sum/indosum_fold0_nusantara_t2t/1.0.0/f779ca1f657fd0562f1534b73b03183ed41ad8f80f4b8003b0a19bd0e0888633. Subsequent calls will reuse this data.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "36b5ec95dba54bdca36569523e6c242b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/3 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Downloading and preparing dataset minang_nl_pmt/minangnlp_mt_nusantara_t2t to /home/samuel/.cache/huggingface/datasets/minang_nl_pmt/minangnlp_mt_nusantara_t2t/1.0.0/d95956ac1d04411db5056a7f3824a31458a4accf72975e82173cdb4686dd1c84...\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "17949647f365450eb50474469f367eeb",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading data: 0.00B [00:00, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "baba513d62164e1b8ea269ac04cbb582",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Generating train split: 0 examples [00:00, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5dd06104d2694abcbd82b76f5eed3615",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Generating test split: 0 examples [00:00, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "42b05eddcf8b47368f02f9acf7945762",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Generating validation split: 0 examples [00:00, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Dataset minang_nl_pmt downloaded and prepared to /home/samuel/.cache/huggingface/datasets/minang_nl_pmt/minangnlp_mt_nusantara_t2t/1.0.0/d95956ac1d04411db5056a7f3824a31458a4accf72975e82173cdb4686dd1c84. Subsequent calls will reuse this data.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "dbce7d0dafd640759889fba3c7b231e9",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/3 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Downloading and preparing dataset news_en_id/news_en_id_nusantara_t2t to /home/samuel/.cache/huggingface/datasets/news_en_id/news_en_id_nusantara_t2t/1.0.0/a5c9608bc6314f138f35d229f18bd181d0f95902da4efb90106a8a3cd02a9f74...\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e285e237f04a46f5b3309e0f495f82df",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Generating train split: 0 examples [00:00, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0f9b37b38ca847d595b92beacf4ad5b3",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Generating validation split: 0 examples [00:00, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d1b3fd269a8e46dfb6637e5eb78e9a95",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Generating test split: 0 examples [00:00, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Dataset news_en_id downloaded and prepared to /home/samuel/.cache/huggingface/datasets/news_en_id/news_en_id_nusantara_t2t/1.0.0/a5c9608bc6314f138f35d229f18bd181d0f95902da4efb90106a8a3cd02a9f74. Subsequent calls will reuse this data.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d485307be92c453b94ea2c4a9bcea6b8",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/3 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Downloading and preparing dataset nusa_xmt/nusax_mt_ace_ind_nusantara_t2t to /home/samuel/.cache/huggingface/datasets/nusa_xmt/nusax_mt_ace_ind_nusantara_t2t/1.0.0/8e5cd18bf2771dbfe991f0c4db45678ec7aaecf94420cd375f211b46495e81cf...\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f5068f04edcc400aa1f7ea643b795d11",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading data:   0%|          | 0.00/354k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a9b8d163319a4c559c442c2f711728bb",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading data:   0%|          | 0.00/70.6k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7bc9f87b67b041dda57c428eb0f41aac",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading data:   0%|          | 0.00/285k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "dd3397580e89490b96f245c73828e66c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Generating train split: 0 examples [00:00, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "99f5e886aac14f519e1507bb4c2f1355",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Generating validation split: 0 examples [00:00, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d2fa13741cc244f9a76969fcaba5e831",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Generating test split: 0 examples [00:00, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Dataset nusa_xmt downloaded and prepared to /home/samuel/.cache/huggingface/datasets/nusa_xmt/nusax_mt_ace_ind_nusantara_t2t/1.0.0/8e5cd18bf2771dbfe991f0c4db45678ec7aaecf94420cd375f211b46495e81cf. Subsequent calls will reuse this data.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4b4be0d5247b42c29e747579bcd9bed2",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/3 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Downloading and preparing dataset nusa_xmt/nusax_mt_ban_ind_nusantara_t2t to /home/samuel/.cache/huggingface/datasets/nusa_xmt/nusax_mt_ban_ind_nusantara_t2t/1.0.0/8e5cd18bf2771dbfe991f0c4db45678ec7aaecf94420cd375f211b46495e81cf...\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0bb961608e294708a82c72da2c397a34",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Generating train split: 0 examples [00:00, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "bde4ca10afb641418c17af2ad48128f5",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Generating validation split: 0 examples [00:00, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "26cc2342ccd244459da5af416ff5d517",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Generating test split: 0 examples [00:00, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Dataset nusa_xmt downloaded and prepared to /home/samuel/.cache/huggingface/datasets/nusa_xmt/nusax_mt_ban_ind_nusantara_t2t/1.0.0/8e5cd18bf2771dbfe991f0c4db45678ec7aaecf94420cd375f211b46495e81cf. Subsequent calls will reuse this data.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "62c90c45870a45789fd496dffc6f7392",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/3 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Downloading and preparing dataset nusa_xmt/nusax_mt_bjn_ind_nusantara_t2t to /home/samuel/.cache/huggingface/datasets/nusa_xmt/nusax_mt_bjn_ind_nusantara_t2t/1.0.0/8e5cd18bf2771dbfe991f0c4db45678ec7aaecf94420cd375f211b46495e81cf...\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0b24f88d4a524a8db75cb2e146942984",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Generating train split: 0 examples [00:00, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2961c31c35cc411d9ef04e71361b0810",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Generating validation split: 0 examples [00:00, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "10c98a119a4246a8af1111c559f62c30",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Generating test split: 0 examples [00:00, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Dataset nusa_xmt downloaded and prepared to /home/samuel/.cache/huggingface/datasets/nusa_xmt/nusax_mt_bjn_ind_nusantara_t2t/1.0.0/8e5cd18bf2771dbfe991f0c4db45678ec7aaecf94420cd375f211b46495e81cf. Subsequent calls will reuse this data.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ca1ff6fd67b64dbca25aca1518ae40c1",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/3 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Downloading and preparing dataset nusa_xmt/nusax_mt_bug_ind_nusantara_t2t to /home/samuel/.cache/huggingface/datasets/nusa_xmt/nusax_mt_bug_ind_nusantara_t2t/1.0.0/8e5cd18bf2771dbfe991f0c4db45678ec7aaecf94420cd375f211b46495e81cf...\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6fe70c44a4cc4192b09495e802e2fcad",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Generating train split: 0 examples [00:00, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7d191a24827b450eb887c09b45fecc57",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Generating validation split: 0 examples [00:00, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f3ec231f10d04c63b3f645bfb3ed29b5",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Generating test split: 0 examples [00:00, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Dataset nusa_xmt downloaded and prepared to /home/samuel/.cache/huggingface/datasets/nusa_xmt/nusax_mt_bug_ind_nusantara_t2t/1.0.0/8e5cd18bf2771dbfe991f0c4db45678ec7aaecf94420cd375f211b46495e81cf. Subsequent calls will reuse this data.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "494e8c65d17b43e3bbf99749a4d06f95",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/3 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Downloading and preparing dataset nusa_xmt/nusax_mt_eng_ind_nusantara_t2t to /home/samuel/.cache/huggingface/datasets/nusa_xmt/nusax_mt_eng_ind_nusantara_t2t/1.0.0/8e5cd18bf2771dbfe991f0c4db45678ec7aaecf94420cd375f211b46495e81cf...\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9a42e6b4c839471e86e9e2c358c3e0b8",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Generating train split: 0 examples [00:00, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b0453a7a7edc4c3a8d6ca15b97efa908",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Generating validation split: 0 examples [00:00, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "de652d4b7999467c893a270486330201",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Generating test split: 0 examples [00:00, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Dataset nusa_xmt downloaded and prepared to /home/samuel/.cache/huggingface/datasets/nusa_xmt/nusax_mt_eng_ind_nusantara_t2t/1.0.0/8e5cd18bf2771dbfe991f0c4db45678ec7aaecf94420cd375f211b46495e81cf. Subsequent calls will reuse this data.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0a0ef781628644b894fae5784eb39533",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/3 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Downloading and preparing dataset nusa_xmt/nusax_mt_ind_ace_nusantara_t2t to /home/samuel/.cache/huggingface/datasets/nusa_xmt/nusax_mt_ind_ace_nusantara_t2t/1.0.0/8e5cd18bf2771dbfe991f0c4db45678ec7aaecf94420cd375f211b46495e81cf...\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "bc11b6ae9bbb4d0a906fd76c1b903a6f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Generating train split: 0 examples [00:00, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b402195dbcd046aca96e6c4be5f63e3c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Generating validation split: 0 examples [00:00, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e1d340d5eb2046e2a6cdab1eb97ef2f2",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Generating test split: 0 examples [00:00, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Dataset nusa_xmt downloaded and prepared to /home/samuel/.cache/huggingface/datasets/nusa_xmt/nusax_mt_ind_ace_nusantara_t2t/1.0.0/8e5cd18bf2771dbfe991f0c4db45678ec7aaecf94420cd375f211b46495e81cf. Subsequent calls will reuse this data.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6fd8d3965247470da22e7d7f52310de0",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/3 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Downloading and preparing dataset nusa_xmt/nusax_mt_ind_ban_nusantara_t2t to /home/samuel/.cache/huggingface/datasets/nusa_xmt/nusax_mt_ind_ban_nusantara_t2t/1.0.0/8e5cd18bf2771dbfe991f0c4db45678ec7aaecf94420cd375f211b46495e81cf...\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ceb625a017c6491aa42312ff79a74484",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Generating train split: 0 examples [00:00, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e7385309c3a44f028a67ef99a4d1a6a2",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Generating validation split: 0 examples [00:00, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1fad734d13a2426cbca92923ef316f51",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Generating test split: 0 examples [00:00, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Dataset nusa_xmt downloaded and prepared to /home/samuel/.cache/huggingface/datasets/nusa_xmt/nusax_mt_ind_ban_nusantara_t2t/1.0.0/8e5cd18bf2771dbfe991f0c4db45678ec7aaecf94420cd375f211b46495e81cf. Subsequent calls will reuse this data.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e7e65f0befd44b7ca15a3c1e802b58d9",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/3 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Downloading and preparing dataset nusa_xmt/nusax_mt_ind_bjn_nusantara_t2t to /home/samuel/.cache/huggingface/datasets/nusa_xmt/nusax_mt_ind_bjn_nusantara_t2t/1.0.0/8e5cd18bf2771dbfe991f0c4db45678ec7aaecf94420cd375f211b46495e81cf...\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "33c1ce9ddbcf42b4b195644e5fcb033c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Generating train split: 0 examples [00:00, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2d899ad02fcc46258d8951cd4d87a983",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Generating validation split: 0 examples [00:00, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b0f01a63a11a481195244cf37458bc24",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Generating test split: 0 examples [00:00, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Dataset nusa_xmt downloaded and prepared to /home/samuel/.cache/huggingface/datasets/nusa_xmt/nusax_mt_ind_bjn_nusantara_t2t/1.0.0/8e5cd18bf2771dbfe991f0c4db45678ec7aaecf94420cd375f211b46495e81cf. Subsequent calls will reuse this data.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4ad707fd5a0a43da8b581691dafd75ff",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/3 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Downloading and preparing dataset nusa_xmt/nusax_mt_ind_bug_nusantara_t2t to /home/samuel/.cache/huggingface/datasets/nusa_xmt/nusax_mt_ind_bug_nusantara_t2t/1.0.0/8e5cd18bf2771dbfe991f0c4db45678ec7aaecf94420cd375f211b46495e81cf...\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b6b382151dd24c57aa9ce13d7e161d9b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Generating train split: 0 examples [00:00, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "450df42a67e6426c81b9152c6ad6a498",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Generating validation split: 0 examples [00:00, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2a3209eadbd04c0680bd0f83b2673339",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Generating test split: 0 examples [00:00, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Dataset nusa_xmt downloaded and prepared to /home/samuel/.cache/huggingface/datasets/nusa_xmt/nusax_mt_ind_bug_nusantara_t2t/1.0.0/8e5cd18bf2771dbfe991f0c4db45678ec7aaecf94420cd375f211b46495e81cf. Subsequent calls will reuse this data.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ee88236a89f64c74bfd9d9a8170364f1",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/3 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Downloading and preparing dataset nusa_xmt/nusax_mt_ind_eng_nusantara_t2t to /home/samuel/.cache/huggingface/datasets/nusa_xmt/nusax_mt_ind_eng_nusantara_t2t/1.0.0/8e5cd18bf2771dbfe991f0c4db45678ec7aaecf94420cd375f211b46495e81cf...\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6abad40569a246918a90d8443e773aa6",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Generating train split: 0 examples [00:00, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "613bf63485e8433f9f3bfb7d4f8baee4",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Generating validation split: 0 examples [00:00, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "43b1f81f9ff6473694f580876bd8d116",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Generating test split: 0 examples [00:00, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Dataset nusa_xmt downloaded and prepared to /home/samuel/.cache/huggingface/datasets/nusa_xmt/nusax_mt_ind_eng_nusantara_t2t/1.0.0/8e5cd18bf2771dbfe991f0c4db45678ec7aaecf94420cd375f211b46495e81cf. Subsequent calls will reuse this data.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e39f074a098a42d4a0472683cc568850",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/3 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Downloading and preparing dataset nusa_xmt/nusax_mt_ind_jav_nusantara_t2t to /home/samuel/.cache/huggingface/datasets/nusa_xmt/nusax_mt_ind_jav_nusantara_t2t/1.0.0/8e5cd18bf2771dbfe991f0c4db45678ec7aaecf94420cd375f211b46495e81cf...\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2634a06127574baf8379f66368629dcf",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Generating train split: 0 examples [00:00, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d452f93aa17247b1a9daefef324e3c30",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Generating validation split: 0 examples [00:00, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "65b8e648f19e4a2dbb2fb7283f02cccd",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Generating test split: 0 examples [00:00, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Dataset nusa_xmt downloaded and prepared to /home/samuel/.cache/huggingface/datasets/nusa_xmt/nusax_mt_ind_jav_nusantara_t2t/1.0.0/8e5cd18bf2771dbfe991f0c4db45678ec7aaecf94420cd375f211b46495e81cf. Subsequent calls will reuse this data.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "444170dc187b450d9703165461388919",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/3 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Downloading and preparing dataset nusa_xmt/nusax_mt_ind_mad_nusantara_t2t to /home/samuel/.cache/huggingface/datasets/nusa_xmt/nusax_mt_ind_mad_nusantara_t2t/1.0.0/8e5cd18bf2771dbfe991f0c4db45678ec7aaecf94420cd375f211b46495e81cf...\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9cc4e4e35ffa4335a12b725f89f5ce08",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Generating train split: 0 examples [00:00, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "464faf0b3d4745448b9f820b9964abf6",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Generating validation split: 0 examples [00:00, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "fe452fda90af4ec192bbf9547a20a819",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Generating test split: 0 examples [00:00, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Dataset nusa_xmt downloaded and prepared to /home/samuel/.cache/huggingface/datasets/nusa_xmt/nusax_mt_ind_mad_nusantara_t2t/1.0.0/8e5cd18bf2771dbfe991f0c4db45678ec7aaecf94420cd375f211b46495e81cf. Subsequent calls will reuse this data.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "df9624a121bb44bda6112a2e83aeff5a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/3 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Downloading and preparing dataset nusa_xmt/nusax_mt_ind_min_nusantara_t2t to /home/samuel/.cache/huggingface/datasets/nusa_xmt/nusax_mt_ind_min_nusantara_t2t/1.0.0/8e5cd18bf2771dbfe991f0c4db45678ec7aaecf94420cd375f211b46495e81cf...\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9963e687ebb845819356e9be4a261796",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Generating train split: 0 examples [00:00, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "21790038c8d345d2afd4881d0ce4f6b7",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Generating validation split: 0 examples [00:00, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2b5f819b5829407b859ca075d40d552b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Generating test split: 0 examples [00:00, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Dataset nusa_xmt downloaded and prepared to /home/samuel/.cache/huggingface/datasets/nusa_xmt/nusax_mt_ind_min_nusantara_t2t/1.0.0/8e5cd18bf2771dbfe991f0c4db45678ec7aaecf94420cd375f211b46495e81cf. Subsequent calls will reuse this data.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "081c7452d81e4749838ad2cea6a8af79",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/3 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Downloading and preparing dataset nusa_xmt/nusax_mt_ind_nij_nusantara_t2t to /home/samuel/.cache/huggingface/datasets/nusa_xmt/nusax_mt_ind_nij_nusantara_t2t/1.0.0/8e5cd18bf2771dbfe991f0c4db45678ec7aaecf94420cd375f211b46495e81cf...\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f308f556cd25465eb5ba2371401f8990",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Generating train split: 0 examples [00:00, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9ff1cb7fc50f41eca1090ba453386608",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Generating validation split: 0 examples [00:00, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "df09c05ff0d148b08f157bdf47a9d3c2",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Generating test split: 0 examples [00:00, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Dataset nusa_xmt downloaded and prepared to /home/samuel/.cache/huggingface/datasets/nusa_xmt/nusax_mt_ind_nij_nusantara_t2t/1.0.0/8e5cd18bf2771dbfe991f0c4db45678ec7aaecf94420cd375f211b46495e81cf. Subsequent calls will reuse this data.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ed5b0aab03b9492f85700a0963b5b80d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/3 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Downloading and preparing dataset nusa_xmt/nusax_mt_ind_sun_nusantara_t2t to /home/samuel/.cache/huggingface/datasets/nusa_xmt/nusax_mt_ind_sun_nusantara_t2t/1.0.0/8e5cd18bf2771dbfe991f0c4db45678ec7aaecf94420cd375f211b46495e81cf...\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "851e926ee8c84e22bf9808e038b613e1",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Generating train split: 0 examples [00:00, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "49d67d740d294042ada13a01c3f1c95d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Generating validation split: 0 examples [00:00, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0d49bf8497eb4827966eac07b283fbea",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Generating test split: 0 examples [00:00, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Dataset nusa_xmt downloaded and prepared to /home/samuel/.cache/huggingface/datasets/nusa_xmt/nusax_mt_ind_sun_nusantara_t2t/1.0.0/8e5cd18bf2771dbfe991f0c4db45678ec7aaecf94420cd375f211b46495e81cf. Subsequent calls will reuse this data.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "bcbc7a22086e46d8914185f30fa48c67",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/3 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Downloading and preparing dataset nusa_xmt/nusax_mt_ind_bbc_nusantara_t2t to /home/samuel/.cache/huggingface/datasets/nusa_xmt/nusax_mt_ind_bbc_nusantara_t2t/1.0.0/8e5cd18bf2771dbfe991f0c4db45678ec7aaecf94420cd375f211b46495e81cf...\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4f7cf6e0b6c74433bb16bfcf695fafc5",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Generating train split: 0 examples [00:00, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "73ef4ff8e440486d996bddd1a824ee3a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Generating validation split: 0 examples [00:00, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "569d93c502bd4586a71b46f45f0e724f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Generating test split: 0 examples [00:00, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Dataset nusa_xmt downloaded and prepared to /home/samuel/.cache/huggingface/datasets/nusa_xmt/nusax_mt_ind_bbc_nusantara_t2t/1.0.0/8e5cd18bf2771dbfe991f0c4db45678ec7aaecf94420cd375f211b46495e81cf. Subsequent calls will reuse this data.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3c1c676cb1ae4051abdadb227aa75cd0",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/3 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Downloading and preparing dataset nusa_xmt/nusax_mt_jav_ind_nusantara_t2t to /home/samuel/.cache/huggingface/datasets/nusa_xmt/nusax_mt_jav_ind_nusantara_t2t/1.0.0/8e5cd18bf2771dbfe991f0c4db45678ec7aaecf94420cd375f211b46495e81cf...\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ec340414a6ec40a1a048a134b954d99c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Generating train split: 0 examples [00:00, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e1a75e126dec442294c99c6d291a24a7",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Generating validation split: 0 examples [00:00, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2f3fc1d167da4fe8aabec617a2e31c0b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Generating test split: 0 examples [00:00, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Dataset nusa_xmt downloaded and prepared to /home/samuel/.cache/huggingface/datasets/nusa_xmt/nusax_mt_jav_ind_nusantara_t2t/1.0.0/8e5cd18bf2771dbfe991f0c4db45678ec7aaecf94420cd375f211b46495e81cf. Subsequent calls will reuse this data.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b073264680174215bd4ac85a250d5820",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/3 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Downloading and preparing dataset nusa_xmt/nusax_mt_mad_ind_nusantara_t2t to /home/samuel/.cache/huggingface/datasets/nusa_xmt/nusax_mt_mad_ind_nusantara_t2t/1.0.0/8e5cd18bf2771dbfe991f0c4db45678ec7aaecf94420cd375f211b46495e81cf...\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d70a04fe78b4418c94ab5f014a6bb76c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Generating train split: 0 examples [00:00, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "bbd41ae2f3a64d7eb1f3334c52ec750b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Generating validation split: 0 examples [00:00, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "26c7f2fab2fd484e88baed37785f4931",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Generating test split: 0 examples [00:00, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Dataset nusa_xmt downloaded and prepared to /home/samuel/.cache/huggingface/datasets/nusa_xmt/nusax_mt_mad_ind_nusantara_t2t/1.0.0/8e5cd18bf2771dbfe991f0c4db45678ec7aaecf94420cd375f211b46495e81cf. Subsequent calls will reuse this data.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "714ddf2f760f422cad310dbd88ea332b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/3 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Downloading and preparing dataset nusa_xmt/nusax_mt_min_ind_nusantara_t2t to /home/samuel/.cache/huggingface/datasets/nusa_xmt/nusax_mt_min_ind_nusantara_t2t/1.0.0/8e5cd18bf2771dbfe991f0c4db45678ec7aaecf94420cd375f211b46495e81cf...\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "73c9f4fcbf354828942ee71f55b91d21",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Generating train split: 0 examples [00:00, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "fbf118ee8432472ab1550b5da80dedc2",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Generating validation split: 0 examples [00:00, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6ee586a0c2a94e12929276714879550f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Generating test split: 0 examples [00:00, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Dataset nusa_xmt downloaded and prepared to /home/samuel/.cache/huggingface/datasets/nusa_xmt/nusax_mt_min_ind_nusantara_t2t/1.0.0/8e5cd18bf2771dbfe991f0c4db45678ec7aaecf94420cd375f211b46495e81cf. Subsequent calls will reuse this data.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b8e532a7593c40a4b7d2023dc0ef151e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/3 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Downloading and preparing dataset nusa_xmt/nusax_mt_nij_ind_nusantara_t2t to /home/samuel/.cache/huggingface/datasets/nusa_xmt/nusax_mt_nij_ind_nusantara_t2t/1.0.0/8e5cd18bf2771dbfe991f0c4db45678ec7aaecf94420cd375f211b46495e81cf...\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b1ca88e175e74d2fad0fe42974f2f079",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Generating train split: 0 examples [00:00, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a27d3aa640294129ab7783e4af40f496",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Generating validation split: 0 examples [00:00, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1498cf6642c1401182cf0d5c4001dce6",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Generating test split: 0 examples [00:00, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Dataset nusa_xmt downloaded and prepared to /home/samuel/.cache/huggingface/datasets/nusa_xmt/nusax_mt_nij_ind_nusantara_t2t/1.0.0/8e5cd18bf2771dbfe991f0c4db45678ec7aaecf94420cd375f211b46495e81cf. Subsequent calls will reuse this data.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d69b86c84007463f8521b09c36c7ba05",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/3 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Downloading and preparing dataset nusa_xmt/nusax_mt_sun_ind_nusantara_t2t to /home/samuel/.cache/huggingface/datasets/nusa_xmt/nusax_mt_sun_ind_nusantara_t2t/1.0.0/8e5cd18bf2771dbfe991f0c4db45678ec7aaecf94420cd375f211b46495e81cf...\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5a218e45414846bfa5a2f75cb3f2a6ec",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Generating train split: 0 examples [00:00, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d58dd4b0999d4a3ba54510a5f665db33",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Generating validation split: 0 examples [00:00, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "bb15ae764cb54ca3afa75e35f634c92d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Generating test split: 0 examples [00:00, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Dataset nusa_xmt downloaded and prepared to /home/samuel/.cache/huggingface/datasets/nusa_xmt/nusax_mt_sun_ind_nusantara_t2t/1.0.0/8e5cd18bf2771dbfe991f0c4db45678ec7aaecf94420cd375f211b46495e81cf. Subsequent calls will reuse this data.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e9b9cf2fc1154ec5bd9f18a8e4bb5c47",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/3 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Downloading and preparing dataset nusa_xmt/nusax_mt_bbc_ind_nusantara_t2t to /home/samuel/.cache/huggingface/datasets/nusa_xmt/nusax_mt_bbc_ind_nusantara_t2t/1.0.0/8e5cd18bf2771dbfe991f0c4db45678ec7aaecf94420cd375f211b46495e81cf...\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a347ed0c19f74c838a821850fcb40914",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Generating train split: 0 examples [00:00, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2deb0a662fff4f61861ab15ab9da006e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Generating validation split: 0 examples [00:00, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0d7a5d09672547afb280f80563f6f5e6",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Generating test split: 0 examples [00:00, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Dataset nusa_xmt downloaded and prepared to /home/samuel/.cache/huggingface/datasets/nusa_xmt/nusax_mt_bbc_ind_nusantara_t2t/1.0.0/8e5cd18bf2771dbfe991f0c4db45678ec7aaecf94420cd375f211b46495e81cf. Subsequent calls will reuse this data.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b5be9a5be5624a1281a5b51ab0614f9e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/3 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Downloading and preparing dataset para_cotta/paracotta_id_nusantara_t2t to /home/samuel/.cache/huggingface/datasets/para_cotta/paracotta_id_nusantara_t2t/1.0.0/4b63c156f44006c55c9c4a8eee3cea966db5c075d4afc914be994f3251ddd8e2...\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "905f9454b5f841188a44919fa54bb7f6",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading data:   0%|          | 0.00/889M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Generating train split: 0 examples [00:00, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Dataset para_cotta downloaded and prepared to /home/samuel/.cache/huggingface/datasets/para_cotta/paracotta_id_nusantara_t2t/1.0.0/4b63c156f44006c55c9c4a8eee3cea966db5c075d4afc914be994f3251ddd8e2. Subsequent calls will reuse this data.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "153f2fb09bb34395bb6a99b56a9f9fb9",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/1 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:datasets.builder:Reusing dataset parallel_su_id (/home/samuel/.cache/huggingface/datasets/parallel_su_id/parallel_su_id_nusantara_t2t/1.0.0/f370015e1234edb5d6820b49517dcd52a1acce9f675b453ce05ffbba2e8b34c0)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "56b49af88a114b00bf45f18ccdcd871f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/1 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Downloading and preparing dataset stif_indonesia/stif_indonesia_nusantara_t2t to /home/samuel/.cache/huggingface/datasets/stif_indonesia/stif_indonesia_nusantara_t2t/1.0.0/e2a7ca266edc3e1906afc3d121b172498d5b6ae74bd39b7b1b46535c58d48255...\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "27b14e67af53471d96eb35e2e5bc3e51",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading data:   0%|          | 0.00/6.89k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4b0232c509154a4fabeefc31c9eaf8dd",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading data:   0%|          | 0.00/7.11k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "943f028e162c4a90bff1faee6ea3f7e0",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading data:   0%|          | 0.00/11.9k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3b97ba748fe4442fba5cca730708919d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading data:   0%|          | 0.00/12.5k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e217a0a601734342aff74f175d8c7f13",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading data:   0%|          | 0.00/55.8k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "65c3677683f145d78704a7bef11eee11",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading data:   0%|          | 0.00/58.6k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Generating test split: 0 examples [00:00, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Generating validation split: 0 examples [00:00, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Generating train split: 0 examples [00:00, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Dataset stif_indonesia downloaded and prepared to /home/samuel/.cache/huggingface/datasets/stif_indonesia/stif_indonesia_nusantara_t2t/1.0.0/e2a7ca266edc3e1906afc3d121b172498d5b6ae74bd39b7b1b46535c58d48255. Subsequent calls will reuse this data.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "421a6754fcc743f191ac648a6d57fd07",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/3 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Downloading and preparing dataset ted_en_id/ted_en_id_nusantara_t2t to /home/samuel/.cache/huggingface/datasets/ted_en_id/ted_en_id_nusantara_t2t/1.0.0/238640abbb8d60a5c5e091a85d78eb1b939b81afab21f385b61834e9137b34b5...\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Generating train split: 0 examples [00:00, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Generating validation split: 0 examples [00:00, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Generating test split: 0 examples [00:00, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Dataset ted_en_id downloaded and prepared to /home/samuel/.cache/huggingface/datasets/ted_en_id/ted_en_id_nusantara_t2t/1.0.0/238640abbb8d60a5c5e091a85d78eb1b939b81afab21f385b61834e9137b34b5. Subsequent calls will reuse this data.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "92a2f10e201d4b8895e6f4beeb6e9972",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/3 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Downloading and preparing dataset ud_id_csui_dataset/ud_id_csui_nusantara_t2t to /home/samuel/.cache/huggingface/datasets/ud_id_csui_dataset/ud_id_csui_nusantara_t2t/1.0.0/617f7a6ffc44867d5cb6ca1b90f92468f5d7eac873c290392071ea76abecd637...\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "83a67934ee8c49a78998596bfd333cef",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading data files:   0%|          | 0/2 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Generating train split: 0 examples [00:00, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Generating test split: 0 examples [00:00, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Dataset ud_id_csui_dataset downloaded and prepared to /home/samuel/.cache/huggingface/datasets/ud_id_csui_dataset/ud_id_csui_nusantara_t2t/1.0.0/617f7a6ffc44867d5cb6ca1b90f92468f5d7eac873c290392071ea76abecd637. Subsequent calls will reuse this data.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "70f6bb4f7edf48a0aeec66e8c3488982",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/2 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Downloading and preparing dataset xl_sum/xl_sum_nusantara_t2t to /home/samuel/.cache/huggingface/datasets/xl_sum/xl_sum_nusantara_t2t/1.0.0/76624c668cf6d1c08c111c12ef243a9d87b01fdff321ec667bd87016705c4a2e...\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Generating train split: 0 examples [00:00, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Generating validation split: 0 examples [00:00, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Generating test split: 0 examples [00:00, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Dataset xl_sum downloaded and prepared to /home/samuel/.cache/huggingface/datasets/xl_sum/xl_sum_nusantara_t2t/1.0.0/76624c668cf6d1c08c111c12ef243a9d87b01fdff321ec667bd87016705c4a2e. Subsequent calls will reuse this data.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "972ef38470cd44b696004c09b1d8754b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/3 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Downloading and preparing dataset x_persona_id/xpersona_id_nusantara_t2t to /home/samuel/.cache/huggingface/datasets/x_persona_id/xpersona_id_nusantara_t2t/1.0.0/421f069a8f9e1c2b6fdfcd287bf16582ef6a4154ed2cfb6c724bf19e0369db8b...\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Generating train split: 0 examples [00:00, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Generating test split: 0 examples [00:00, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Generating validation split: 0 examples [00:00, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Dataset x_persona_id downloaded and prepared to /home/samuel/.cache/huggingface/datasets/x_persona_id/xpersona_id_nusantara_t2t/1.0.0/421f069a8f9e1c2b6fdfcd287bf16582ef6a4154ed2cfb6c724bf19e0369db8b. Subsequent calls will reuse this data.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4f01b8ee093a4359976389d39da94ec4",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/3 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "text_generation_datasets = {\n",
    "    helper.config.name: helper.load_dataset() for helper in conhelps.filtered(lambda x: x.config.name in TEXT_GENERATION_TASKS)\n",
    "}"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8ab2d2cf-55fc-4d1a-a13a-ed7e9332244b",
   "metadata": {},
   "source": [
    "# Count TC"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "f8e4dc7f-cce7-420d-b370-bd67eac64a17",
   "metadata": {},
   "outputs": [],
   "source": [
    "dset_count = {}\n",
    "for dset_name in text_classification_datasets.keys():\n",
    "    dset_count[dset_name] = {'train': 0, 'validation': 0, 'test': 0}\n",
    "    for split in text_classification_datasets[dset_name].keys():\n",
    "        dset_count[dset_name][split] = len(text_classification_datasets[dset_name][split])\n",
    "tc_df = pd.DataFrame(dset_count)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "652dd702-9db4-4d2a-97b5-635598b8d527",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "casa_nusantara_text_multi                        1080.0\n",
       "emot_nusantara_text                              4401.0\n",
       "emotcmt_nusantara_text                            582.0\n",
       "emotion_id_opinion_nusantara_text                7080.0\n",
       "hoasa_nusantara_text_multi                       2854.0\n",
       "id_abusive_nusantara_text                        2016.0\n",
       "id_clickbait_nusantara_text                     15000.0\n",
       "id_google_play_review_nusantara_text            10040.0\n",
       "id_google_play_review_posneg_nusantara_text     10040.0\n",
       "id_hatespeech_nusantara_text                      713.0\n",
       "id_multilabel_hs_nusantara_text_multi           13169.0\n",
       "id_stance_nusantara_pairs                         337.0\n",
       "id_sts_nusantara_pairs_score                    12901.0\n",
       "imdb_jv_nusantara_text                         100000.0\n",
       "indolem_ntp_nusantara_pairs                     33528.0\n",
       "indolem_sentiment_nusantara_text                 5048.0\n",
       "indonli_nusantara_pairs                         17709.0\n",
       "jadi_ide_nusantara_text                         16498.0\n",
       "netifier_nusantara_text_multi                    7773.0\n",
       "nusax_senti_ace_nusantara_text                   1000.0\n",
       "nusax_senti_ban_nusantara_text                   1000.0\n",
       "nusax_senti_bjn_nusantara_text                   1000.0\n",
       "nusax_senti_bug_nusantara_text                   1000.0\n",
       "nusax_senti_eng_nusantara_text                   1000.0\n",
       "nusax_senti_ind_nusantara_text                   1000.0\n",
       "nusax_senti_jav_nusantara_text                   1000.0\n",
       "nusax_senti_mad_nusantara_text                   1000.0\n",
       "nusax_senti_min_nusantara_text                   1000.0\n",
       "nusax_senti_nij_nusantara_text                   1000.0\n",
       "nusax_senti_sun_nusantara_text                   1000.0\n",
       "nusax_senti_bbc_nusantara_text                   1000.0\n",
       "nusax_senti_nusantara_text                      12000.0\n",
       "sentiment_nathasa_review_nusantara_text        124263.0\n",
       "smsa_nusantara_text                             12760.0\n",
       "dtype: float64"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tc_df.sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "4934c679-8038-4595-b224-7cc4a2680a0e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "train           239623.0\n",
       "validation       16338.0\n",
       "test            115831.0\n",
       "unsupervised     50000.0\n",
       "dtype: float64"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tc_df.T.sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "id": "3dfc4b86-cdf0-4c20-ac08-f1b88b6a3251",
   "metadata": {},
   "outputs": [],
   "source": [
    "tc_metadata = {}\n",
    "for row in tc_df.T.itertuples():\n",
    "    if row.test != 0:\n",
    "        tc_metadata[row.Index] = ('test', row.test)\n",
    "    elif row.validation != 0:\n",
    "        tc_metadata[row.Index] = ('validation', row.validation)\n",
    "    elif row.train != 0:\n",
    "        tc_metadata[row.Index] = ('train', row.train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "id": "5b13f95f-f36a-437a-80b7-a8de63f820f5",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>casa_nusantara_text_multi</th>\n",
       "      <td>test</td>\n",
       "      <td>180.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>emot_nusantara_text</th>\n",
       "      <td>test</td>\n",
       "      <td>440.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>emotcmt_nusantara_text</th>\n",
       "      <td>test</td>\n",
       "      <td>582.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>emotion_id_opinion_nusantara_text</th>\n",
       "      <td>train</td>\n",
       "      <td>7080.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>hoasa_nusantara_text_multi</th>\n",
       "      <td>test</td>\n",
       "      <td>286.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>id_abusive_nusantara_text</th>\n",
       "      <td>train</td>\n",
       "      <td>2016.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>id_clickbait_nusantara_text</th>\n",
       "      <td>train</td>\n",
       "      <td>15000.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>id_google_play_review_nusantara_text</th>\n",
       "      <td>validation</td>\n",
       "      <td>3012.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>id_google_play_review_posneg_nusantara_text</th>\n",
       "      <td>validation</td>\n",
       "      <td>3012.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>id_hatespeech_nusantara_text</th>\n",
       "      <td>train</td>\n",
       "      <td>713.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>id_multilabel_hs_nusantara_text_multi</th>\n",
       "      <td>train</td>\n",
       "      <td>13169.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>id_stance_nusantara_pairs</th>\n",
       "      <td>train</td>\n",
       "      <td>337.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>id_sts_nusantara_pairs_score</th>\n",
       "      <td>test</td>\n",
       "      <td>2580.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>imdb_jv_nusantara_text</th>\n",
       "      <td>test</td>\n",
       "      <td>25000.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>indolem_ntp_nusantara_pairs</th>\n",
       "      <td>test</td>\n",
       "      <td>7560.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>indolem_sentiment_nusantara_text</th>\n",
       "      <td>test</td>\n",
       "      <td>1011.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>indonli_nusantara_pairs</th>\n",
       "      <td>test</td>\n",
       "      <td>5183.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>jadi_ide_nusantara_text</th>\n",
       "      <td>train</td>\n",
       "      <td>16498.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>netifier_nusantara_text_multi</th>\n",
       "      <td>test</td>\n",
       "      <td>778.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>nusax_senti_ace_nusantara_text</th>\n",
       "      <td>test</td>\n",
       "      <td>400.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>nusax_senti_ban_nusantara_text</th>\n",
       "      <td>test</td>\n",
       "      <td>400.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>nusax_senti_bjn_nusantara_text</th>\n",
       "      <td>test</td>\n",
       "      <td>400.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>nusax_senti_bug_nusantara_text</th>\n",
       "      <td>test</td>\n",
       "      <td>400.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>nusax_senti_eng_nusantara_text</th>\n",
       "      <td>test</td>\n",
       "      <td>400.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>nusax_senti_ind_nusantara_text</th>\n",
       "      <td>test</td>\n",
       "      <td>400.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>nusax_senti_jav_nusantara_text</th>\n",
       "      <td>test</td>\n",
       "      <td>400.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>nusax_senti_mad_nusantara_text</th>\n",
       "      <td>test</td>\n",
       "      <td>400.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>nusax_senti_min_nusantara_text</th>\n",
       "      <td>test</td>\n",
       "      <td>400.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>nusax_senti_nij_nusantara_text</th>\n",
       "      <td>test</td>\n",
       "      <td>400.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>nusax_senti_sun_nusantara_text</th>\n",
       "      <td>test</td>\n",
       "      <td>400.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>nusax_senti_bbc_nusantara_text</th>\n",
       "      <td>test</td>\n",
       "      <td>400.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>nusax_senti_nusantara_text</th>\n",
       "      <td>test</td>\n",
       "      <td>4800.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>sentiment_nathasa_review_nusantara_text</th>\n",
       "      <td>test</td>\n",
       "      <td>62131.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>smsa_nusantara_text</th>\n",
       "      <td>test</td>\n",
       "      <td>500.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                      0        1\n",
       "casa_nusantara_text_multi                          test    180.0\n",
       "emot_nusantara_text                                test    440.0\n",
       "emotcmt_nusantara_text                             test    582.0\n",
       "emotion_id_opinion_nusantara_text                 train   7080.0\n",
       "hoasa_nusantara_text_multi                         test    286.0\n",
       "id_abusive_nusantara_text                         train   2016.0\n",
       "id_clickbait_nusantara_text                       train  15000.0\n",
       "id_google_play_review_nusantara_text         validation   3012.0\n",
       "id_google_play_review_posneg_nusantara_text  validation   3012.0\n",
       "id_hatespeech_nusantara_text                      train    713.0\n",
       "id_multilabel_hs_nusantara_text_multi             train  13169.0\n",
       "id_stance_nusantara_pairs                         train    337.0\n",
       "id_sts_nusantara_pairs_score                       test   2580.0\n",
       "imdb_jv_nusantara_text                             test  25000.0\n",
       "indolem_ntp_nusantara_pairs                        test   7560.0\n",
       "indolem_sentiment_nusantara_text                   test   1011.0\n",
       "indonli_nusantara_pairs                            test   5183.0\n",
       "jadi_ide_nusantara_text                           train  16498.0\n",
       "netifier_nusantara_text_multi                      test    778.0\n",
       "nusax_senti_ace_nusantara_text                     test    400.0\n",
       "nusax_senti_ban_nusantara_text                     test    400.0\n",
       "nusax_senti_bjn_nusantara_text                     test    400.0\n",
       "nusax_senti_bug_nusantara_text                     test    400.0\n",
       "nusax_senti_eng_nusantara_text                     test    400.0\n",
       "nusax_senti_ind_nusantara_text                     test    400.0\n",
       "nusax_senti_jav_nusantara_text                     test    400.0\n",
       "nusax_senti_mad_nusantara_text                     test    400.0\n",
       "nusax_senti_min_nusantara_text                     test    400.0\n",
       "nusax_senti_nij_nusantara_text                     test    400.0\n",
       "nusax_senti_sun_nusantara_text                     test    400.0\n",
       "nusax_senti_bbc_nusantara_text                     test    400.0\n",
       "nusax_senti_nusantara_text                         test   4800.0\n",
       "sentiment_nathasa_review_nusantara_text            test  62131.0\n",
       "smsa_nusantara_text                                test    500.0"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\n",
    "\n",
    "pd.DataFrame(tc_metadata).T"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "id": "33b4cc46-ee91-4300-b023-f6321a6d5b61",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "176668.0"
      ]
     },
     "execution_count": 56,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.DataFrame(tc_metadata).T[1].sum()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ab3b5e1f-f93b-46bf-9dce-e62f4206e943",
   "metadata": {},
   "source": [
    "# Count TG"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "id": "47edc71d-4e30-462a-a480-56a0fad4e3c0",
   "metadata": {},
   "outputs": [],
   "source": [
    "dset_count = {}\n",
    "for dset_name in text_generation_datasets.keys():\n",
    "    dset_count[dset_name] = {'train': 0, 'validation': 0, 'test': 0}\n",
    "    for split in text_generation_datasets[dset_name].keys():\n",
    "        dset_count[dset_name][split] = len(text_generation_datasets[dset_name][split])\n",
    "tg_df = pd.DataFrame(dset_count)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "id": "2a832017-a92e-41b3-aa22-83157adae28e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "bible_en_id_nusantara_t2t                  31078\n",
       "bible_jv_id_nusantara_t2t                   7958\n",
       "bible_su_id_nusantara_t2t                   7957\n",
       "id_panl_bppt_nusantara_t2t                 24021\n",
       "id_qqp_nusantara_t2t                      149011\n",
       "indo_general_mt_en_id_nusantara_t2t      1825716\n",
       "indo_religious_mt_en_id_nusantara_t2t     589368\n",
       "indosum_fold0_nusantara_t2t                18774\n",
       "minangnlp_mt_nusantara_t2t                 16371\n",
       "news_en_id_nusantara_t2t                   42369\n",
       "nusax_mt_ace_ind_nusantara_t2t              1000\n",
       "nusax_mt_ban_ind_nusantara_t2t              1000\n",
       "nusax_mt_bjn_ind_nusantara_t2t              1000\n",
       "nusax_mt_bug_ind_nusantara_t2t              1000\n",
       "nusax_mt_eng_ind_nusantara_t2t              1000\n",
       "nusax_mt_ind_ace_nusantara_t2t              1000\n",
       "nusax_mt_ind_ban_nusantara_t2t              1000\n",
       "nusax_mt_ind_bjn_nusantara_t2t              1000\n",
       "nusax_mt_ind_bug_nusantara_t2t              1000\n",
       "nusax_mt_ind_eng_nusantara_t2t              1000\n",
       "nusax_mt_ind_jav_nusantara_t2t              1000\n",
       "nusax_mt_ind_mad_nusantara_t2t              1000\n",
       "nusax_mt_ind_min_nusantara_t2t              1000\n",
       "nusax_mt_ind_nij_nusantara_t2t              1000\n",
       "nusax_mt_ind_sun_nusantara_t2t              1000\n",
       "nusax_mt_ind_bbc_nusantara_t2t              1000\n",
       "nusax_mt_jav_ind_nusantara_t2t              1000\n",
       "nusax_mt_mad_ind_nusantara_t2t              1000\n",
       "nusax_mt_min_ind_nusantara_t2t              1000\n",
       "nusax_mt_nij_ind_nusantara_t2t              1000\n",
       "nusax_mt_sun_ind_nusantara_t2t              1000\n",
       "nusax_mt_bbc_ind_nusantara_t2t              1000\n",
       "paracotta_id_nusantara_t2t               6000000\n",
       "parallel_su_id_nusantara_t2t                3616\n",
       "stif_indonesia_nusantara_t2t                2499\n",
       "ted_en_id_nusantara_t2t                    93262\n",
       "ud_id_csui_nusantara_t2t                    1030\n",
       "xl_sum_nusantara_t2t                       47802\n",
       "xpersona_id_nusantara_t2t                 131673\n",
       "dtype: int64"
      ]
     },
     "execution_count": 58,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tg_df.sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "id": "28eab7fc-75ab-4b9d-b14e-01c94b132d2c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "train         8925867\n",
       "validation      44585\n",
       "test            44053\n",
       "dtype: int64"
      ]
     },
     "execution_count": 59,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tg_df.T.sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "id": "2ab0d4b5-aa36-4405-84f5-fbd79ac5e74c",
   "metadata": {},
   "outputs": [],
   "source": [
    "tg_metadata = {}\n",
    "for row in tg_df.T.itertuples():\n",
    "    if row.test != 0:\n",
    "        tg_metadata[row.Index] = ('test', row.test)\n",
    "    elif row.validation != 0:\n",
    "        tg_metadata[row.Index] = ('validation', row.validation)\n",
    "    elif row.train != 0:\n",
    "        tg_metadata[row.Index] = ('train', row.train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "id": "1d8beb7c-d289-4d16-ba12-a30d8290f2a4",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>bible_en_id_nusantara_t2t</th>\n",
       "      <td>test</td>\n",
       "      <td>4661</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>bible_jv_id_nusantara_t2t</th>\n",
       "      <td>test</td>\n",
       "      <td>1193</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>bible_su_id_nusantara_t2t</th>\n",
       "      <td>test</td>\n",
       "      <td>1193</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>id_panl_bppt_nusantara_t2t</th>\n",
       "      <td>train</td>\n",
       "      <td>24021</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>id_qqp_nusantara_t2t</th>\n",
       "      <td>validation</td>\n",
       "      <td>14927</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>indo_general_mt_en_id_nusantara_t2t</th>\n",
       "      <td>test</td>\n",
       "      <td>2000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>indo_religious_mt_en_id_nusantara_t2t</th>\n",
       "      <td>test</td>\n",
       "      <td>4824</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>indosum_fold0_nusantara_t2t</th>\n",
       "      <td>test</td>\n",
       "      <td>3762</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>minangnlp_mt_nusantara_t2t</th>\n",
       "      <td>test</td>\n",
       "      <td>3200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>news_en_id_nusantara_t2t</th>\n",
       "      <td>test</td>\n",
       "      <td>1954</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>nusax_mt_ace_ind_nusantara_t2t</th>\n",
       "      <td>test</td>\n",
       "      <td>400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>nusax_mt_ban_ind_nusantara_t2t</th>\n",
       "      <td>test</td>\n",
       "      <td>400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>nusax_mt_bjn_ind_nusantara_t2t</th>\n",
       "      <td>test</td>\n",
       "      <td>400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>nusax_mt_bug_ind_nusantara_t2t</th>\n",
       "      <td>test</td>\n",
       "      <td>400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>nusax_mt_eng_ind_nusantara_t2t</th>\n",
       "      <td>test</td>\n",
       "      <td>400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>nusax_mt_ind_ace_nusantara_t2t</th>\n",
       "      <td>test</td>\n",
       "      <td>400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>nusax_mt_ind_ban_nusantara_t2t</th>\n",
       "      <td>test</td>\n",
       "      <td>400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>nusax_mt_ind_bjn_nusantara_t2t</th>\n",
       "      <td>test</td>\n",
       "      <td>400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>nusax_mt_ind_bug_nusantara_t2t</th>\n",
       "      <td>test</td>\n",
       "      <td>400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>nusax_mt_ind_eng_nusantara_t2t</th>\n",
       "      <td>test</td>\n",
       "      <td>400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>nusax_mt_ind_jav_nusantara_t2t</th>\n",
       "      <td>test</td>\n",
       "      <td>400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>nusax_mt_ind_mad_nusantara_t2t</th>\n",
       "      <td>test</td>\n",
       "      <td>400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>nusax_mt_ind_min_nusantara_t2t</th>\n",
       "      <td>test</td>\n",
       "      <td>400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>nusax_mt_ind_nij_nusantara_t2t</th>\n",
       "      <td>test</td>\n",
       "      <td>400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>nusax_mt_ind_sun_nusantara_t2t</th>\n",
       "      <td>test</td>\n",
       "      <td>400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>nusax_mt_ind_bbc_nusantara_t2t</th>\n",
       "      <td>test</td>\n",
       "      <td>400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>nusax_mt_jav_ind_nusantara_t2t</th>\n",
       "      <td>test</td>\n",
       "      <td>400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>nusax_mt_mad_ind_nusantara_t2t</th>\n",
       "      <td>test</td>\n",
       "      <td>400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>nusax_mt_min_ind_nusantara_t2t</th>\n",
       "      <td>test</td>\n",
       "      <td>400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>nusax_mt_nij_ind_nusantara_t2t</th>\n",
       "      <td>test</td>\n",
       "      <td>400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>nusax_mt_sun_ind_nusantara_t2t</th>\n",
       "      <td>test</td>\n",
       "      <td>400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>nusax_mt_bbc_ind_nusantara_t2t</th>\n",
       "      <td>test</td>\n",
       "      <td>400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>paracotta_id_nusantara_t2t</th>\n",
       "      <td>train</td>\n",
       "      <td>6000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>parallel_su_id_nusantara_t2t</th>\n",
       "      <td>train</td>\n",
       "      <td>3616</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>stif_indonesia_nusantara_t2t</th>\n",
       "      <td>test</td>\n",
       "      <td>363</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ted_en_id_nusantara_t2t</th>\n",
       "      <td>test</td>\n",
       "      <td>3179</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ud_id_csui_nusantara_t2t</th>\n",
       "      <td>test</td>\n",
       "      <td>374</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>xl_sum_nusantara_t2t</th>\n",
       "      <td>test</td>\n",
       "      <td>4780</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>xpersona_id_nusantara_t2t</th>\n",
       "      <td>test</td>\n",
       "      <td>3770</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                0        1\n",
       "bible_en_id_nusantara_t2t                    test     4661\n",
       "bible_jv_id_nusantara_t2t                    test     1193\n",
       "bible_su_id_nusantara_t2t                    test     1193\n",
       "id_panl_bppt_nusantara_t2t                  train    24021\n",
       "id_qqp_nusantara_t2t                   validation    14927\n",
       "indo_general_mt_en_id_nusantara_t2t          test     2000\n",
       "indo_religious_mt_en_id_nusantara_t2t        test     4824\n",
       "indosum_fold0_nusantara_t2t                  test     3762\n",
       "minangnlp_mt_nusantara_t2t                   test     3200\n",
       "news_en_id_nusantara_t2t                     test     1954\n",
       "nusax_mt_ace_ind_nusantara_t2t               test      400\n",
       "nusax_mt_ban_ind_nusantara_t2t               test      400\n",
       "nusax_mt_bjn_ind_nusantara_t2t               test      400\n",
       "nusax_mt_bug_ind_nusantara_t2t               test      400\n",
       "nusax_mt_eng_ind_nusantara_t2t               test      400\n",
       "nusax_mt_ind_ace_nusantara_t2t               test      400\n",
       "nusax_mt_ind_ban_nusantara_t2t               test      400\n",
       "nusax_mt_ind_bjn_nusantara_t2t               test      400\n",
       "nusax_mt_ind_bug_nusantara_t2t               test      400\n",
       "nusax_mt_ind_eng_nusantara_t2t               test      400\n",
       "nusax_mt_ind_jav_nusantara_t2t               test      400\n",
       "nusax_mt_ind_mad_nusantara_t2t               test      400\n",
       "nusax_mt_ind_min_nusantara_t2t               test      400\n",
       "nusax_mt_ind_nij_nusantara_t2t               test      400\n",
       "nusax_mt_ind_sun_nusantara_t2t               test      400\n",
       "nusax_mt_ind_bbc_nusantara_t2t               test      400\n",
       "nusax_mt_jav_ind_nusantara_t2t               test      400\n",
       "nusax_mt_mad_ind_nusantara_t2t               test      400\n",
       "nusax_mt_min_ind_nusantara_t2t               test      400\n",
       "nusax_mt_nij_ind_nusantara_t2t               test      400\n",
       "nusax_mt_sun_ind_nusantara_t2t               test      400\n",
       "nusax_mt_bbc_ind_nusantara_t2t               test      400\n",
       "paracotta_id_nusantara_t2t                  train  6000000\n",
       "parallel_su_id_nusantara_t2t                train     3616\n",
       "stif_indonesia_nusantara_t2t                 test      363\n",
       "ted_en_id_nusantara_t2t                      test     3179\n",
       "ud_id_csui_nusantara_t2t                     test      374\n",
       "xl_sum_nusantara_t2t                         test     4780\n",
       "xpersona_id_nusantara_t2t                    test     3770"
      ]
     },
     "execution_count": 63,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.DataFrame(tg_metadata).T"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "id": "43c609c0-af3d-4e6e-b23f-4f038bf70809",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "6086617"
      ]
     },
     "execution_count": 72,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.DataFrame(tg_metadata).T[1].sum()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8c42454c-d1a3-4b4d-9378-147a5cdb79ab",
   "metadata": {},
   "source": [
    "# Save Count Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "id": "a3acb817-1297-4211-9118-ef7d13a1e11a",
   "metadata": {},
   "outputs": [],
   "source": [
    "pd.DataFrame(tg_metadata).T.rename({0: 'split', 1: 'num_data'}, axis=1).reset_index().to_csv('tg_dataset.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "id": "9ce12d8f-ca92-439d-be87-6e13e4c491c4",
   "metadata": {},
   "outputs": [],
   "source": [
    "pd.DataFrame(tc_metadata).T.rename({0: 'split', 1: 'num_data'}, axis=1).reset_index().to_csv('tc_dataset.csv', index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "eb87bac0-017c-4684-986d-ed5043d016d1",
   "metadata": {
    "tags": []
   },
   "source": [
    "# Test Experiment"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 124,
   "id": "bdbf8213-c890-4fc0-b52f-de8701a69a04",
   "metadata": {},
   "outputs": [],
   "source": [
    "cfg_to_split_name = pd.DataFrame(tc_metadata).T[0].to_dict()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 132,
   "id": "a7b7075c-8886-4f36-8b15-12ed044a3438",
   "metadata": {},
   "outputs": [],
   "source": [
    "from transformers import AutoTokenizer, AutoModelForCausalLM\n",
    "import numpy as np\n",
    "import torch\n",
    "import torch.nn.functional as F\n",
    "from tqdm import tqdm\n",
    "\n",
    "from sklearn.metrics import accuracy_score, f1_score, recall_score, precision_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 133,
   "id": "33805f30-940a-41bc-9315-641a6af0633a",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_logprobs(tokenizer, model, prompt):\n",
    "    inputs = tokenizer(prompt, return_tensors=\"pt\")\n",
    "    input_ids, output_ids = inputs[\"input_ids\"], inputs[\"input_ids\"][:, 1:]\n",
    "    outputs = model(**inputs, labels=input_ids)\n",
    "    logits = outputs.logits\n",
    "    logprobs = torch.gather(F.log_softmax(logits, dim=2), 2, output_ids.unsqueeze(2))\n",
    "    return logprobs\n",
    "\n",
    "# Zero-shot evaluation for the Choice of Plausible Alternatives (COPA) task.\n",
    "# A return value of 0 indicates that the first alternative is more plausible,\n",
    "# while 1 indicates that the second alternative is more plausible.\n",
    "def prompt_eval(tokenizer, model, prompt, alternatives):\n",
    "    probs = []\n",
    "    for alt in alternatives:\n",
    "        probs.append(get_logprobs(prompt + \"\\n\" + f\"Choose between ({', '.join(alternatives)})\" + \"\\n\", alt).sum())\n",
    "    return np.argmax(probs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "77c1d41c-e06a-4a5d-9569-f00ff9fc703a",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load Model\n",
    "# facebook/xglm-564M, facebook/xglm-1.7B, facebook/xglm-2.9B, facebook/xglm-4.5B, facebook/xglm-7.5B\n",
    "tokenizer = AutoTokenizer.from_pretrained(\"facebook/xglm-7.5B\")\n",
    "model = AutoModelForCausalLM.from_pretrained(\"facebook/xglm-7.5B\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "313d692e-e0af-4914-b187-dd164d2d2cce",
   "metadata": {},
   "outputs": [],
   "source": [
    "eval_results = {}\n",
    "for key, dset in text_classification_datasets.items():\n",
    "    print(f'Evaluating `{key}`')\n",
    "    # Get Idx to Label\n",
    "    if '_text_multi' in key:\n",
    "        labels = dset[cfg_to_split_name[key]].info.features['labels'].feature.names\n",
    "        idx2label = {i: lab for i, lab in enumerate(dset[cfg_to_split_name[key]].info.features['labels'].feature.names)}\n",
    "        # Skip for now\n",
    "        continue\n",
    "    elif '_text' in key:\n",
    "        labels = dset[cfg_to_split_name[key]].info.features['label'].names\n",
    "        idx2label = {i: lab for i, lab in enumerate(dset[cfg_to_split_name[key]].info.features['label'].names)}\n",
    "    elif '_pairs_score' in key:\n",
    "        # Skip for now\n",
    "        continue\n",
    "    elif '_pairs' in key:\n",
    "        labels = dset[cfg_to_split_name[key]].info.features['label'].names\n",
    "        idx2label = {i: lab for i, lab in enumerate(dset[cfg_to_split_name[key]].info.features['label'].names)}\n",
    "    else:\n",
    "        raise ValueError('Unknown Dataset Type')\n",
    "\n",
    "    # Iterate Dataset\n",
    "    preds = []\n",
    "    golds = []\n",
    "    for row in tqdm(dset[cfg_to_split_name[key]]):\n",
    "        if '_text' in key:\n",
    "            prompt = row.text\n",
    "        elif '_pairs' in key:\n",
    "            prompt = row.text_1 + \"\\n\" + row.text_2\n",
    "        \n",
    "        preds.append(prompt_eval(tokenizer, model, prompt, labels))\n",
    "        golds.append(row.label)       \n",
    "        \n",
    "    metrics = {}\n",
    "    metrics[\"ACC\"] = accuracy_score(list_label, list_hyp)\n",
    "    metrics[\"F1-macro\"] = f1_score(list_label, list_hyp, average='macro')\n",
    "    metrics[\"REC-macro\"] = recall_score(list_label, list_hyp, average='macro')\n",
    "    metrics[\"PRE-macro\"] = precision_score(list_label, list_hyp, average='macro')\n",
    "    metrics[\"F1-micro\"] = f1_score(list_label, list_hyp, average='micro')\n",
    "    metrics[\"REC-micro\"] = recall_score(list_label, list_hyp, average='micro')\n",
    "    metrics[\"PRE-micro\"] = precision_score(list_label, list_hyp, average='micro')\n",
    "    \n",
    "    eval_results[key] = metrics\n",
    "    break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 129,
   "id": "5280a747-672b-4a40-a8da-4ab2daf3016e",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "922562d8-773f-4f43-8253-377b768a6762",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "396\n"
     ]
    }
   ],
   "source": [
    "conhelps = NusantaraConfigHelper()\n",
    "nusantara_helpers = conhelps.filtered(\n",
    "    lambda x: x.is_nusantara_schema and not x.is_resource\n",
    ")\n",
    "print(len(nusantara_helpers))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "9530d443-5d5b-4555-a8ab-fbb8c84716fd",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "33\n"
     ]
    }
   ],
   "source": [
    "conhelps = NusantaraConfigHelper()\n",
    "nusantara_helpers = conhelps.filtered(\n",
    "    lambda x: x.is_nusantara_schema \n",
    "        and not x.is_resource \n",
    "        and (x.config.name.endswith('_text') or x.config.name.endswith('_pairs'))\n",
    ")\n",
    "print(len(nusantara_helpers))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "06b9ed53-d4f1-4f67-896c-6bb8a491586d",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "242\n"
     ]
    }
   ],
   "source": [
    "conhelps = NusantaraConfigHelper()\n",
    "nusantara_helpers = conhelps.filtered(\n",
    "    lambda x: x.is_nusantara_schema \n",
    "        and not x.is_resource \n",
    "        and x.config.name.endswith('_t2t')\n",
    ")\n",
    "print(len(nusantara_helpers))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "id": "406fbc7d-bf15-4c06-812c-551d2a870eec",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0                AM2iCo\n",
       "2                  CASA\n",
       "8      COCO Captions ID\n",
       "9                  CORD\n",
       "10            CoVoST 2 \n",
       "             ...       \n",
       "160               WReTe\n",
       "161              X-FACT\n",
       "162               XCOPA\n",
       "163              XL-Sum\n",
       "164         XPersona Id\n",
       "Name: name, Length: 72, dtype: object"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "meta_df = pd.read_csv('https://docs.google.com/spreadsheets/d/17o83IvWxmtGLYridZis0nEprHhsZIMeFtHGtXV35h6M/export?format=csv&gid=879729812', skiprows=1)\n",
    "meta_df = meta_df.rename({\n",
    "    'No.': 'id', 'Name': 'name', 'Subsets': 'subsets', 'Link': 'source_link', 'Description': 'description',\n",
    "    'HF Link': 'hf_link', 'License': 'license', 'Year': 'year', 'Collection Style': 'collection_style',\n",
    "    'Language': 'language', 'Dialect': 'dialect', 'Domain': 'domain', 'Form': 'modality', 'Tasks': 'tasks',\n",
    "    'Volume': 'volume', 'Unit': 'unit', 'Ethical Risks': 'ethical_risk', 'Provider': 'provider',\n",
    "    'Paper Title': 'paper_title', 'Paper Link': 'paper_link', 'Access': 'access', 'Derived From': 'derived_from', \n",
    "    'Test Split': 'is_splitted', 'Notes': 'notes', 'Dataloader': 'dataloader', 'Implemented': 'implemented'\n",
    "}, axis=1)\n",
    "meta_df['is_splitted'] = meta_df['is_splitted'].apply(lambda x: True if x =='Yes' else False)\n",
    "# [\n",
    "#  'No.', 'Name', 'Subsets', 'Link', 'HF Link', 'License', 'Year',\n",
    "#  'Language', 'Dialect', 'Domain', 'Form', 'Collection Style',\n",
    "#  'Description', 'Volume', 'Unit', 'Ethical Risks', 'Provider',\n",
    "#  'Paper Title', 'Paper Link', 'Access', 'Derived From', 'Tasks',\n",
    "#  'Test Split', 'Notes', 'Dataloader', 'Implemented'\n",
    "# ]\n",
    "meta_df.loc[meta_df.is_splitted, 'name']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "id": "72389399-4675-4b11-8f8e-ea6d579e3b88",
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset_name = meta_df.loc[meta_df.is_splitted, 'dataloader'].tolist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "id": "58be9e82-3230-499f-9eea-b4ec963843d0",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['bible_en_id',\n",
       " 'bible_jv_id',\n",
       " 'bible_su_id',\n",
       " 'covost2_ind_eng',\n",
       " 'covost2_eng_ind',\n",
       " 'id_panl_bppt',\n",
       " 'id_qqp',\n",
       " 'id_wiki_parallel_jav_ind',\n",
       " 'id_wiki_parallel_min_ind',\n",
       " 'id_wiki_parallel_sun_ind',\n",
       " 'indo_general_mt_en_id',\n",
       " 'indo_religious_mt_en_id',\n",
       " 'indosum_fold0',\n",
       " 'indosum_fold1',\n",
       " 'indosum_fold2',\n",
       " 'indosum_fold3',\n",
       " 'indosum_fold4',\n",
       " 'korpus_nusantara_ind_jav',\n",
       " 'korpus_nusantara_ind_day',\n",
       " 'korpus_nusantara_ind_bug',\n",
       " 'korpus_nusantara_ind_sun',\n",
       " 'korpus_nusantara_ind_mad',\n",
       " 'korpus_nusantara_ind_bin',\n",
       " 'korpus_nusantara_ind_bbc',\n",
       " 'korpus_nusantara_ind_khek',\n",
       " 'korpus_nusantara_ind_msa',\n",
       " 'korpus_nusantara_ind_min',\n",
       " 'korpus_nusantara_ind_tiociu',\n",
       " 'korpus_nusantara_jav_ind',\n",
       " 'korpus_nusantara_day_ind',\n",
       " 'korpus_nusantara_bug_ind',\n",
       " 'korpus_nusantara_sun_ind',\n",
       " 'korpus_nusantara_mad_ind',\n",
       " 'korpus_nusantara_bin_ind',\n",
       " 'korpus_nusantara_bbc_ind',\n",
       " 'korpus_nusantara_khek_ind',\n",
       " 'korpus_nusantara_msa_ind',\n",
       " 'korpus_nusantara_min_ind',\n",
       " 'korpus_nusantara_tiociu_ind',\n",
       " 'minangnlp_mt',\n",
       " 'multilexnorm',\n",
       " 'news_en_id',\n",
       " 'nllb_seed_ace',\n",
       " 'nllb_seed_bjn',\n",
       " 'nllb_seed_bug',\n",
       " 'nusax_mt_ace_ind',\n",
       " 'nusax_mt_ban_ind',\n",
       " 'nusax_mt_bjn_ind',\n",
       " 'nusax_mt_bug_ind',\n",
       " 'nusax_mt_eng_ind',\n",
       " 'nusax_mt_ind_ace',\n",
       " 'nusax_mt_ind_ban',\n",
       " 'nusax_mt_ind_bjn',\n",
       " 'nusax_mt_ind_bug',\n",
       " 'nusax_mt_ind_eng',\n",
       " 'nusax_mt_ind_jav',\n",
       " 'nusax_mt_ind_mad',\n",
       " 'nusax_mt_ind_min',\n",
       " 'nusax_mt_ind_nij',\n",
       " 'nusax_mt_ind_sun',\n",
       " 'nusax_mt_ind_bbc',\n",
       " 'nusax_mt_jav_ind',\n",
       " 'nusax_mt_mad_ind',\n",
       " 'nusax_mt_min_ind',\n",
       " 'nusax_mt_nij_ind',\n",
       " 'nusax_mt_sun_ind',\n",
       " 'nusax_mt_bbc_ind',\n",
       " 'paracotta_id',\n",
       " 'parallel_su_id',\n",
       " 'stif_indonesia',\n",
       " 'talpco_eng_ind',\n",
       " 'talpco_ind_eng',\n",
       " 'talpco_ind_jpn',\n",
       " 'talpco_ind_kor',\n",
       " 'talpco_ind_myn',\n",
       " 'talpco_ind_tha',\n",
       " 'talpco_ind_vie',\n",
       " 'talpco_ind_zsm',\n",
       " 'talpco_jpn_ind',\n",
       " 'talpco_kor_ind',\n",
       " 'talpco_myn_ind',\n",
       " 'talpco_tha_ind',\n",
       " 'talpco_vie_ind',\n",
       " 'talpco_zsm_ind',\n",
       " 'ted_en_id',\n",
       " 'ud_id_csui',\n",
       " 'xl_sum',\n",
       " 'xpersona_id']"
      ]
     },
     "execution_count": 55,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "[\n",
    "    helper.config.name.split('_nusantara_t2t')[0] for helper in nusantara_helpers\n",
    "            if (('nusax_mt' not in helper.config.name and 'talpco' not in helper.config.name) or 'ind' in helper.config.name)\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "80a723a1-4036-4010-bbe6-c69f5c762afb",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python (env_nusa_exp)",
   "language": "python",
   "name": "env_nusa_exp"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}