2-choose_MT_model.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "env: CUDA_DEVICE_ORDER=PCI_BUS_ID\n",
      "env: CUDA_VISIBLE_DEVICES=5\n"
     ]
    }
   ],
   "source": [
    "%env CUDA_DEVICE_ORDER=PCI_BUS_ID\n",
    "%env CUDA_VISIBLE_DEVICES=5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from conllu import parse\n",
    "import pandas as pd\n",
    "import os\n",
    "\n",
    "# Define the language code, used in the file names\n",
    "lang_code = \"ES-PV-eu\"\n",
    "\n",
    "# Main path\n",
    "main_path = \"/home/tajak/Parlamint-translation\"\n",
    "\n",
    "# Check whether the path to the folder with conllu files is ok\n",
    "path = \"{}/Source-data/ParlaMint-{}.conllu/ParlaMint-{}.conllu\".format(main_path, lang_code, lang_code)\n",
    "\n",
    "# Define other paths\n",
    "extracted_dataframe_path = \"{}/results/{}/ParlaMint-{}-extracted-source-data.csv\".format(main_path, lang_code, lang_code)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We need to translate the following corpora into English:\n",
    "- Belgian (BE) - in Dutch and French, separate CONLL-us (!!). French: [\"fr\", \"itc\",\"roa\"], Dutch: [\"nl\", \"gem\", \"gmw\"]\n",
    "- Bulgarian (BG): [\"bg\", \"sla\", \"zls\"]\n",
    "- Croatian (HR) -> \"zls\"\n",
    "- Czech (CZ) -> \"cs\"\n",
    "- Danish (DK): [\"da\", \"gmq\", \"gem\"]\n",
    "- Dutch (NL): [\"nl\", \"gem\", \"gmw\"]\n",
    "- French (FR): [\"fr\", \"itc\",\"roa\"]\n",
    "- Hungarian (HU): [\"hu\", \"fiu\", \"urj\"]\n",
    "- Icelandic (IS): [\"is\",\"gmq\", \"gem\"]\n",
    "- Italian (IT): [\"it\", \"roa\", \"itc\"]\n",
    "- Latvian (LV): [\"lv\", \"bat\"]\n",
    "- Lithuanian (LT): only \"bat\"\n",
    "- Polish (PL): [\"pl\", \"sla\", \"zlw\"]\n",
    "- Slovenian (SI) - We will use \"Slavic MT\" based on the results of the manual analysis --> \"sla\"\n",
    "- Spanish? (ES): [\"es\", \"roa\", \"itc\"]\n",
    "- Turkish (TR): [\"tr\", \"trk\" ]\n",
    "- Austrian (AT): [\"de\", \"gem\", \"gmw\"]\n",
    "- Basque (ES-PV): [\"eu\", \"mul\"]\n",
    "- Bosnian (BA): [\"sla\", \"zls\"]\n",
    "- Catalan (ES-CT): [\"ca\", \"roa\", \"itc\"]\n",
    "- Estonian (EE): [\"et\", \"urj\", \"fiu\"]\n",
    "- Finnish (FI): [\"fi\", \"urj\", \"fiu\"]\n",
    "- Galician (ES-GA): [\"gl\", \"roa\", \"itc\"]\n",
    "- Greek (GR):  [\"el\", \"grk\"]\n",
    "- Norwegian (NO): [\"gem\", \"gmq\"]\n",
    "- Portuguese (PT): [\"pt\", \"roa\", \"itc\"]\n",
    "- Romanian (RO): [\"roa\", \"itc\"]\n",
    "- Serbian (RS): [\"sla\", \"zls\"]\n",
    "- Swedish (SE): [\"sv\", \"gmq\", \"gem\"]\n",
    "- Ukrainian (UA): [\"uk\", \"sla\", \"zle\"]\n",
    "- Russian (present in UA): [\"ru\", \"sla\", \"zle\"]\n",
    "\n",
    "Languages with only one option: Lithuanian (\"LT\": \"bat\")"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Explanation of language codes:\n",
    "- sla = Slavic\n",
    "- zls = South Slavic\n",
    "- zlw = West Slavic\n",
    "- zle = East Slavic\n",
    "- gmq = North Germanic\n",
    "- gem = Germanic\n",
    "- gmw = West Germanic\n",
    "- roa = Romance\n",
    "- itc = Italic\n",
    "- bat = Baltic\n",
    "- trk = Turkic\n",
    "- urj = Uralic\n",
    "- fiu = Finno-Ugrian"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/tajak/Parlamint-translation/parlamint_env/lib/python3.8/site-packages/tqdm/auto.py:22: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
      "  from .autonotebook import tqdm as notebook_tqdm\n",
      "2023-10-06 11:16:21.141383: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations:  AVX2 FMA\n",
      "To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.\n",
      "2023-10-06 11:16:21.845452: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory\n",
      "2023-10-06 11:16:21.845514: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory\n",
      "2023-10-06 11:16:21.845519: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "All languages with target_lang=de. I.e., we can translate from these languages to English (en).\n",
      "['aav', 'af', 'alv', 'ar', 'az', 'bat', 'bcl', 'bem', 'ber', 'bg', 'bi', 'bnt', 'bzs', 'ca', 'ceb', 'cel', 'chk', 'cpf', 'crs', 'cs', 'cus', 'cy', 'da', 'de', 'dra', 'ee', 'efi', 'el', 'eo', 'es', 'et', 'eu', 'euq', 'fi', 'fj', 'fr', 'ga', 'gaa', 'gil', 'gl', 'grk', 'guw', 'gv', 'ha', 'he', 'hi', 'hil', 'ho', 'ht', 'hu', 'hy', 'id', 'ig', 'ilo', 'is', 'iso', 'it', 'jap', 'kg', 'kj', 'kqn', 'kwn', 'kwy', 'lg', 'ln', 'loz', 'lu', 'lua', 'lue', 'lun', 'luo', 'lus', 'map', 'mfe', 'mg', 'mh', 'mk', 'mkh', 'ml', 'mos', 'mr', 'mt', 'mul', 'ng', 'nic', 'niu', 'nl', 'nso', 'ny', 'nyk', 'om', 'pag', 'pap', 'phi', 'pis', 'pon', 'poz', 'pqe', 'pqw', 'rn', 'rnd', 'ro', 'roa', 'ru', 'run', 'rw', 'sal', 'sg', 'sit', 'sk', 'sm', 'sn', 'sq', 'ss', 'st', 'sv', 'sw', 'swc', 'tdt', 'ti', 'tiv', 'tl', 'tll', 'tn', 'to', 'toi', 'tpi', 'trk', 'ts', 'tut', 'tvl', 'tw', 'ty', 'uk', 'umb', 'ur', 'vi', 'xh', 'zh']\n"
     ]
    }
   ],
   "source": [
    "from easynmt import EasyNMT\n",
    "\n",
    "# Define the model\n",
    "model = EasyNMT('opus-mt')\n",
    "\n",
    "print(\"\\n\\nAll languages with target_lang=de. I.e., we can translate from these languages to English (en).\")\n",
    "print(model.get_languages(source_lang='en'))\n",
    "\n"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Create sample (if df has not been extracted yet)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "No. of files: 515.\n",
      "Number of words in the corpora: 59823\n",
      "Dataframe saved as /home/tajak/Parlamint-translation/results/HU/ParlaMint-HU-extracted-sample.csv\n",
      "|        | file_path                                               | file                           | sentence_id                    | text                   | tokenized_text          | proper_nouns   |    length |\n",
      "|:-------|:--------------------------------------------------------|:-------------------------------|:-------------------------------|:-----------------------|:------------------------|:---------------|----------:|\n",
      "| count  | 3606                                                    | 3606                           | 3606                           | 3606                   | 3606                    | 3606           | 3606      |\n",
      "| unique | 1                                                       | 1                              | 3606                           | 2938                   | 2938                    | 407            |  nan      |\n",
      "| top    | ParlaMint-HU.conllu/2017/ParlaMint-HU_2017-02-20.conllu | ParlaMint-HU_2017-02-20.conllu | ParlaMint-HU_2017-02-20.seg1.1 | Tisztelt Országgyűlés! | Tisztelt Országgyűlés ! | {}             |  nan      |\n",
      "| freq   | 3606                                                    | 3606                           | 1                              | 95                     | 95                      | 3065           |  nan      |\n",
      "| mean   | nan                                                     | nan                            | nan                            | nan                    | nan                     | nan            |   16.5899 |\n",
      "| std    | nan                                                     | nan                            | nan                            | nan                    | nan                     | nan            |   14.1432 |\n",
      "| min    | nan                                                     | nan                            | nan                            | nan                    | nan                     | nan            |    1      |\n",
      "| 25%    | nan                                                     | nan                            | nan                            | nan                    | nan                     | nan            |    5      |\n",
      "| 50%    | nan                                                     | nan                            | nan                            | nan                    | nan                     | nan            |   13      |\n",
      "| 75%    | nan                                                     | nan                            | nan                            | nan                    | nan                     | nan            |   23      |\n",
      "| max    | nan                                                     | nan                            | nan                            | nan                    | nan                     | nan            |  133      |\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "|    | file_path                                               | file                           | sentence_id                    | text                                                                                                           | tokenized_text                                                                                                   | proper_nouns                                                                                           |   length |\n",
      "|---:|:--------------------------------------------------------|:-------------------------------|:-------------------------------|:---------------------------------------------------------------------------------------------------------------|:-----------------------------------------------------------------------------------------------------------------|:-------------------------------------------------------------------------------------------------------|---------:|\n",
      "|  0 | ParlaMint-HU.conllu/2017/ParlaMint-HU_2017-02-20.conllu | ParlaMint-HU_2017-02-20.conllu | ParlaMint-HU_2017-02-20.seg1.1 | Tisztelt Képviselőtársaim!                                                                                     | Tisztelt Képviselőtársaim !                                                                                      | {}                                                                                                     |        2 |\n",
      "|  1 | ParlaMint-HU.conllu/2017/ParlaMint-HU_2017-02-20.conllu | ParlaMint-HU_2017-02-20.conllu | ParlaMint-HU_2017-02-20.seg1.2 | Az Országgyűlés tavaszi ülésszakának 1. ülésnapját megnyitom.                                                  | Az Országgyűlés tavaszi ülésszakának 1. ülésnapját megnyitom .                                                   | {}                                                                                                     |        7 |\n",
      "|  2 | ParlaMint-HU.conllu/2017/ParlaMint-HU_2017-02-20.conllu | ParlaMint-HU_2017-02-20.conllu | ParlaMint-HU_2017-02-20.seg1.3 | Tájékoztatom önöket, hogy az ülés vezetésében Szűcs Lajos és Hiszékeny Dezső jegyző urak lesznek segítségemre. | Tájékoztatom önöket , hogy az ülés vezetésében Szűcs Lajos és Hiszékeny Dezső jegyző urak lesznek segítségemre . | {7: ['Szűcs', 'Szűcs'], 8: ['Lajos', 'Lajos'], 10: ['Hiszékeny', 'Hiszékeny'], 11: ['Dezső', 'Dezső']} |       15 |\n",
      "|  3 | ParlaMint-HU.conllu/2017/ParlaMint-HU_2017-02-20.conllu | ParlaMint-HU_2017-02-20.conllu | ParlaMint-HU_2017-02-20.seg2.1 | Köszöntöm kedves vendégeinket és mindenkit, aki figyelemmel kíséri a munkánkat.                                | Köszöntöm kedves vendégeinket és mindenkit , aki figyelemmel kíséri a munkánkat .                                | {}                                                                                                     |       10 |\n",
      "|  4 | ParlaMint-HU.conllu/2017/ParlaMint-HU_2017-02-20.conllu | ParlaMint-HU_2017-02-20.conllu | ParlaMint-HU_2017-02-20.seg3.1 | Tisztelt Országgyűlés!                                                                                         | Tisztelt Országgyűlés !                                                                                          | {1: ['Országgyűlés', 'Országgyűlés']}                                                                  |        2 |\n",
      "\n",
      "\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "\n",
    "# Define the path to the folder with conllu files is ok:\n",
    "path = \"ParlaMint-{}.conllu\".format(lang_code)\n",
    "\n",
    "# ------------NO CHANGING OF THE CODE NEEDED FROM NOW ONWARDS--------------\n",
    "\n",
    "# Create a folder with results for this language, e.g. results/CZ\n",
    "os.mkdir(\"/home/tajak/Parlamint-translation/results/{}\".format(lang_code))\n",
    "\n",
    "# Create (manually) a \"temp\" folder inside the results/CZ\n",
    "os.mkdir(\"/home/tajak/Parlamint-translation/results/{}/temp\".format(lang_code))\n",
    "\n",
    "# Define final path\n",
    "extracted_dataframe_path = \"{}/results/{}/ParlaMint-{}-extracted-sample.csv\".format(main_path, lang_code, lang_code)\n",
    "\n",
    "# Extract a list with paths to conllu files and a list with their names\n",
    "parl_list = []\n",
    "file_name_list = []\n",
    "\n",
    "for dir1 in os.listdir(path):\n",
    "    full_path = os.path.join(path, dir1)\n",
    "    if os.path.isdir(full_path):\n",
    "        current = os.listdir(full_path)\n",
    "        # Keep only files with parliamentary sessions:\n",
    "        for file in current:\n",
    "            if \"ParlaMint-{}_\".format(lang_code) in file:\n",
    "                if \".conllu\" in file:\n",
    "                    final_path = \"{}/{}\".format(full_path, file)\n",
    "                    parl_list.append(final_path)\n",
    "                    file_name_list.append(file)\n",
    "\n",
    "# See how many files we have:\n",
    "print(\"No. of files: {}.\".format(len(parl_list)))\n",
    "\n",
    "\n",
    "# Extract only one file\n",
    "def conllu_to_df(parl_list, file_name_list, extracted_dataframe_path):\n",
    "\t\"\"\"\n",
    "\tTake the conllu files and extract relevant information. Save everything in a DataFrame.\n",
    "\n",
    "\tArgs:\n",
    "\t- parl_list: list of documents with their entire paths to be included (see step above).\n",
    "\t- file_name_list: list of names of the files (see step above)\n",
    "\t- extracted_dataframe_path: path to the output file\n",
    "\t\"\"\"\n",
    "\tfrom conllu import parse\n",
    "\timport pandas as pd\n",
    "\n",
    "\t# Create an empty df\n",
    "\tdf = pd.DataFrame({\"file_path\": [\"\"],\"file\": [\"\"], \"sentence_id\": [\"\"], \"text\": [\"\"], \"tokenized_text\": [\"\"], \"proper_nouns\": [\"\"]})\n",
    "\n",
    "\t# Check whether there are any problems with parsing the documents\n",
    "\t\"\"\"\n",
    "\t\n",
    "\terror_count = 0\n",
    "\tproblematic_doc_list = []\n",
    "\n",
    "\tfor doc in parl_list:\n",
    "\t\ttry:\n",
    "\t\t\t# Open the file\n",
    "\t\t\tdata = open(\"{}\".format(doc), \"r\").read()\n",
    "\n",
    "\t\t\tsentences = parse(data)\n",
    "\t\texcept:\n",
    "\t\t\terror_count += 1\n",
    "\t\t\tproblematic_doc_list.append(doc)\n",
    "\n",
    "\tprint(error_count)\n",
    "\tprint(problematic_doc_list)\n",
    "\t\"\"\"\n",
    "\t# Parse the data with CONLL-u parser\n",
    "\tfor doc in parl_list:\n",
    "\t\t# Open the file\n",
    "\t\tdata = open(\"{}\".format(doc), \"r\").read()\n",
    "\t\t\n",
    "\t\tsentences = parse(data)\n",
    "\n",
    "\t\tsentence_id_list = []\n",
    "\t\ttext_list = []\n",
    "\t\ttokenized_text_list = []\n",
    "\t\tproper_noun_list = []\n",
    "\n",
    "\t\tfor sentence in sentences:\n",
    "\t\t\t# Find sentence ids\n",
    "\t\t\tcurrent_sentence_id = sentence.metadata[\"sent_id\"]\n",
    "\t\t\tsentence_id_list.append(current_sentence_id)\n",
    "\n",
    "\t\t\t# Find text - if texts consists of multiword tokens, these tokens will appear as they are,\n",
    "\t\t\t# not separated into subwords\n",
    "\t\t\tcurrent_text = sentence.metadata[\"text\"]\n",
    "\t\t\ttext_list.append(current_text)\n",
    "\n",
    "\t\t\t# Create a string out of tokens\n",
    "\t\t\tcurrent_token_list = []\n",
    "\t\t\tword_dict = {}\n",
    "\n",
    "\t\t\tfor token in sentence:\n",
    "\t\t\t\t# Find multiword tokens and take their NER\n",
    "\t\t\t\tif type(token[\"id\"]) != int:\n",
    "\t\t\t\t\tmultiword_ner = token[\"misc\"][\"NER\"]\n",
    "\t\t\t\t\n",
    "\t\t\t\telse:\n",
    "\t\t\t\t# Append to the tokenized text tokens that are not multiword tokens\n",
    "\t\t\t\t# (we append subtokens to the tokenized texts, not multiword tokens)\n",
    "\t\t\t\t\tcurrent_token_list.append(token[\"form\"])\n",
    "\t\t\t\t\t\n",
    "\n",
    "\t\t\t\t\t# Create a list of NE annotations with word indices.\n",
    "\t\t\t\t\t# I'll substract one from the word index,\n",
    "\t\t\t\t\t# because indexing in the CONLLU file starts with 1, not 0\n",
    "\t\t\t\t\tcurrent_index = int(token[\"id\"]) - 1\n",
    "\n",
    "\t\t\t\t\t# If the word does not have NER annotation,\n",
    "\t\t\t\t\t# take the annotation from the multiword token\n",
    "\t\t\t\t\tif token[\"misc\"] is None:\n",
    "\t\t\t\t\t\tcurrent_ner = multiword_ner\n",
    "\t\t\t\t\telse:\n",
    "\t\t\t\t\t\tcurrent_ner = token[\"misc\"][\"NER\"]\n",
    "\n",
    "\t\t\t\t\t# Add information on the lemma if the NE is personal name\n",
    "\t\t\t\t\tif \"PER\" in current_ner:\n",
    "\t\t\t\t\t\tword_dict[current_index] = [token[\"form\"], token[\"lemma\"]]\n",
    "\n",
    "\t\t\tproper_noun_list.append(word_dict)\n",
    "\n",
    "\t\t\tcurrent_string = \" \".join(current_token_list)\n",
    "\n",
    "\t\t\ttokenized_text_list.append(current_string)\n",
    "\n",
    "\t\t\n",
    "\t\tnew_df = pd.DataFrame({\"sentence_id\": sentence_id_list, \"text\": text_list, \"tokenized_text\": tokenized_text_list, \"proper_nouns\": proper_noun_list})\n",
    "\n",
    "\t\tnew_df[\"file_path\"] = doc\n",
    "\n",
    "\t\t# Get the file name\n",
    "\t\tfile_name = file_name_list[parl_list.index(doc)]\n",
    "\t\tnew_df[\"file\"] = file_name\n",
    "\n",
    "\t\t# Merge df to the previous df\n",
    "\t\tdf = pd.concat([df, new_df])\n",
    "\t\n",
    "\t# Reset index\n",
    "\tdf = df.reset_index(drop=True)\n",
    "\n",
    "\t# Remove the first row\n",
    "\tdf = df.drop([0], axis=\"index\")\n",
    "\n",
    "\t# Reset index\n",
    "\tdf = df.reset_index(drop=True)\n",
    "\n",
    "\t# Add information on length\n",
    "\tdf[\"length\"] = df[\"text\"].str.split().str.len()\n",
    "\n",
    "\tprint(\"Number of words in the corpora: {}\".format(df[\"length\"].sum()))\n",
    "\n",
    "\t# Save the dataframe\n",
    "\tdf.to_csv(\"{}\".format(extracted_dataframe_path), sep=\"\\t\")\n",
    "\n",
    "\tprint(\"Dataframe saved as {}\".format(extracted_dataframe_path))\n",
    "\t\n",
    "\t# Show the results\n",
    "\tprint(df.describe(include=\"all\").to_markdown())\n",
    "\n",
    "\tprint(\"\\n\\n\\n\")\n",
    "\n",
    "\tprint(df.head().to_markdown())\n",
    "\n",
    "\tprint(\"\\n\\n\\n\")\n",
    "\t\n",
    "\treturn df\n",
    "\n",
    "#Extract information from the conllu files: extract only one file\n",
    "df = conllu_to_df(parl_list[:1], file_name_list[:1], extracted_dataframe_path)\n"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Choose model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "def choose_model(lang_code, extracted_dataframe_path):\n",
    "\t\"\"\"\n",
    "\tCompare a small sample of translations of all OPUS-MT models that are available\n",
    "\tfor the language, to decide which one to use. The function prints out a dataframe with all translations of the sample and saves it as ParlaMint-{lang_code}-sample-model-comparison.csv.\n",
    "\n",
    "\tArgs:\n",
    "\t- lang_code: the lang code that is used in the names of the files, it should be the same as for extract_text()\n",
    "\t- extracted_dataframe_path: path to the final output of 1-conllu-to-df.py\n",
    "\t\"\"\"\n",
    "\timport pandas as pd\n",
    "\timport regex as re\n",
    "\tfrom easynmt import EasyNMT\n",
    "\tfrom IPython.display import display\n",
    "\t\n",
    "\tlang_models_dict = {\"BG\": [\"bg\", \"sla\", \"zls\"], \"HR\": [\"zls\", \"sla\"], \"CZ\": [\"cs\", \"sla\", \"zlw\" ], \"DK\": [\"da\", \"gmq\", \"gem\"], \"NL\": [\"nl\", \"gem\", \"gmw\"], \"FR\": [\"fr\", \"itc\",\"roa\"], \"BE-fr\": [\"fr\", \"itc\", \"roa\"], \"BE-nl\": [\"nl\", \"gem\", \"gmw\"], \"HU\": [\"hu\", \"fiu\", \"urj\"], \"IS\": [\"is\",\"gmq\", \"gem\"], \"IT\": [\"it\", \"roa\", \"itc\"], \"LV\": [\"lv\",\"bat\"], \"LT\": [\"bat\"], \"PL\": [\"pl\", \"sla\", \"zlw\"], \"SI\": [\"sla\"], \"ES\": [\"es\", \"roa\", \"itc\"], \"TR\": [\"tr\", \"trk\" ], \"AT\": [\"de\", \"gem\", \"gmw\"], \"ES-PV-eu\": [\"eu\", \"mul\"], \"BA\": [\"sla\", \"zls\"], \"ES-CT\": [\"ca\", \"roa\", \"itc\"], \"EE\": [\"et\", \"urj\", \"fiu\"], \"FI\": [\"fi\", \"urj\", \"fiu\"], \"ES-GA\": [\"gl\", \"roa\", \"itc\"], \"GR\": [\"grk\"], \"NO\": [\"gem\", \"gmq\"], \"PT\": [\"roa\", \"itc\"], \"RO\":[\"roa\", \"itc\"], \"RS\": [\"zls\", \"sla\"], \"SE\": [\"sv\", \"gmq\", \"gem\"], \"UA\":[\"uk\", \"sla\", \"zle\"], \"UA_ru\":[\"ru\", \"sla\", \"zle\"], \"BE\": [\"nl\", \"gem\", \"gmw\"]}\n",
    "\n",
    "\n",
    "\t# Open the file, created in the previous step\n",
    "\tdf = pd.read_csv(\"{}\".format(extracted_dataframe_path), sep=\"\\t\", index_col=0, na_filter = False)\n",
    "\n",
    "\t# Define the model\n",
    "\tmodel = EasyNMT('opus-mt')\n",
    "\n",
    "\tprint(\"Entire corpus has {} sentences and {} words.\".format(df[\"text\"].count(), df[\"length\"].sum()))\n",
    "\n",
    "\tif lang_code == \"UA\":\n",
    "\t\t# Create a sample, making sure that it is in Ukrainian language and that the file is big enough (has 30 sentences)\n",
    "\t\tdf_ua = df[df.file == \"ParlaMint-UA_2013-06-18-m1-uk.conllu\"][:30]\n",
    "\n",
    "\t\tprint(\"Sample files has {} sentences and {} words.\".format(df_ua[\"text\"].count(), df_ua[\"length\"].sum()))\n",
    "\n",
    "\t\t# Create a list of sentences from the df\n",
    "\t\tsentence_list = df_ua.text.to_list()\n",
    "\n",
    "\t\t# Translate the sample using all available models for this language\n",
    "\t\tfor opus_lang_code in lang_models_dict[lang_code]:\n",
    "\t\t\ttranslation_list = model.translate(sentence_list, source_lang = \"{}\".format(opus_lang_code), target_lang='en')\n",
    "\n",
    "\t\t\t# Add the translations to the df\n",
    "\t\t\tdf_ua[\"translation-{}\".format(opus_lang_code)] = translation_list\n",
    "\t\t\n",
    "\t\tdf_ua = df_ua.drop(columns=[\"file\", \"sentence_id\", \"tokenized_text\", \"proper_nouns\", \"length\"])\n",
    "\n",
    "\t\t# Save the df\n",
    "\t\tdf_ua.to_csv(\"/home/tajak/Parlamint-translation/results/{}/ParlaMint-{}-uk-sample-model-comparison.csv\".format(lang_code, lang_code))\n",
    "\n",
    "\t\tprint(\"The file is saved as/home/tajak/Parlamint-translation/results/{}/ParlaMint-{}-uk-sample-model-comparison.csv. \".format(lang_code, lang_code))\n",
    "\n",
    "\t\t# Create also a Russian sample\n",
    "\t\t# Create a sample, making sure that it is in Russian language and that the file is big enough (has 30 sentences)\n",
    "\t\tdf_ru = df[df.file == 'ParlaMint-UA_2013-05-24-m0-ru.conllu'][:30]\n",
    "\n",
    "\t\tprint(\"Russian sample files has {} sentences and {} words.\".format(df_ru[\"text\"].count(), df_ru[\"length\"].sum()))\n",
    "\n",
    "\t\t# Create a list of sentences from the df\n",
    "\t\tru_sentence_list = df_ru.text.to_list()\n",
    "\n",
    "\t\t# Translate the sample using all available models for this language\n",
    "\t\tfor opus_lang_code in lang_models_dict[\"UA_ru\"]:\n",
    "\t\t\tru_translation_list = model.translate(ru_sentence_list, source_lang = \"{}\".format(opus_lang_code), target_lang='en')\n",
    "\n",
    "\t\t\t# Add the translations to the df\n",
    "\t\t\tdf_ru[\"translation-{}\".format(opus_lang_code)] = ru_translation_list\n",
    "\t\t\n",
    "\t\tdf_ru = df_ru.drop(columns=[\"file\", \"sentence_id\", \"tokenized_text\", \"proper_nouns\", \"length\"])\n",
    "\n",
    "\t\t# Save the df\n",
    "\t\tdf_ru.to_csv(\"/home/tajak/Parlamint-translation/results/{}/ParlaMint-{}-ru-sample-model-comparison.csv\".format(lang_code, lang_code))\n",
    "\n",
    "\t\tprint(\"The file is saved as/home/tajak/Parlamint-translation/results/{}/ParlaMint-{}-ru-sample-model-comparison.csv. \".format(lang_code, lang_code))\n",
    "\n",
    "\telif lang_code == \"ES-CT\":\n",
    "\t\t\t# Create a sample, making sure that it is in Catalan language and that the file is big enough (has 30 sentences)\n",
    "\t\tdf_ua = df[df.file == \"ParlaMint-ES-CT_2017-07-13-3902-ca.conllu\"][:30]\n",
    "\n",
    "\t\tprint(\"Sample files has {} sentences and {} words.\".format(df_ua[\"text\"].count(), df_ua[\"length\"].sum()))\n",
    "\n",
    "\t\t# Create a list of sentences from the df\n",
    "\t\tsentence_list = df_ua.text.to_list()\n",
    "\n",
    "\t\t# Translate the sample using all available models for this language\n",
    "\t\tfor opus_lang_code in lang_models_dict[lang_code]:\n",
    "\t\t\ttranslation_list = model.translate(sentence_list, source_lang = \"{}\".format(opus_lang_code), target_lang='en')\n",
    "\n",
    "\t\t\t# Add the translations to the df\n",
    "\t\t\tdf_ua[\"translation-{}\".format(opus_lang_code)] = translation_list\n",
    "\t\t\n",
    "\t\tdf_ua = df_ua.drop(columns=[\"file\", \"sentence_id\", \"tokenized_text\", \"proper_nouns\", \"length\"])\n",
    "\n",
    "\t\t# Save the df\n",
    "\t\tdf_ua.to_csv(\"/home/tajak/Parlamint-translation/results/{}/ParlaMint-{}-ct-sample-model-comparison.csv\".format(lang_code, lang_code))\n",
    "\n",
    "\t\tprint(\"The file is saved as/home/tajak/Parlamint-translation/results/{}/ParlaMint-{}-ct-sample-model-comparison.csv. \".format(lang_code, lang_code))\n",
    "\n",
    "\t\t# Create also a Spanish sample\n",
    "\t\t# Create a sample, making sure that it is in Spanish language and that the file is big enough (has 30 sentences)\n",
    "\t\tdf_ru = df[df.file == 'ParlaMint-ES-CT_2017-10-10-4301-es.conllu'][:30]\n",
    "\n",
    "\t\tprint(\"Spanish sample files has {} sentences and {} words.\".format(df_ru[\"text\"].count(), df_ru[\"length\"].sum()))\n",
    "\n",
    "\t\t# Create a list of sentences from the df\n",
    "\t\tru_sentence_list = df_ru.text.to_list()\n",
    "\n",
    "\t\t# Translate the sample using all available models for this language\n",
    "\t\tfor opus_lang_code in lang_models_dict[\"ES\"]:\n",
    "\t\t\tru_translation_list = model.translate(ru_sentence_list, source_lang = \"{}\".format(opus_lang_code), target_lang='en')\n",
    "\n",
    "\t\t\t# Add the translations to the df\n",
    "\t\t\tdf_ru[\"translation-{}\".format(opus_lang_code)] = ru_translation_list\n",
    "\t\t\n",
    "\t\tdf_ru = df_ru.drop(columns=[\"file\", \"sentence_id\", \"tokenized_text\", \"proper_nouns\", \"length\"])\n",
    "\n",
    "\t\t# Save the df\n",
    "\t\tdf_ru.to_csv(\"/home/tajak/Parlamint-translation/results/{}/ParlaMint-{}-es-sample-model-comparison.csv\".format(lang_code, lang_code))\n",
    "\n",
    "\t\tprint(\"The file is saved as/home/tajak/Parlamint-translation/results/{}/ParlaMint-{}-es-sample-model-comparison.csv. \".format(lang_code, lang_code))\n",
    "\telse:\n",
    "\t\t# Create a smaller sample - just a couple of sentences from one file\n",
    "\t\tdf = df[df.file == list(df[\"file\"].unique())[0]][:30]\n",
    "\n",
    "\t\tprint(\"Sample files has {} sentences and {} words.\".format(df[\"text\"].count(), df[\"length\"].sum()))\n",
    "\n",
    "\t\t# Create a list of sentences from the df\n",
    "\t\tsentence_list = df.text.to_list()\n",
    "\n",
    "\t\t# Translate the sample using all available models for this language\n",
    "\t\tfor opus_lang_code in lang_models_dict[lang_code]:\n",
    "\t\t\ttranslation_list = model.translate(sentence_list, source_lang = \"{}\".format(opus_lang_code), target_lang='en')\n",
    "\n",
    "\t\t\t# Add the translations to the df\n",
    "\t\t\tdf[\"translation-{}\".format(opus_lang_code)] = translation_list\n",
    "\t\t\n",
    "\t\tdf = df.drop(columns=[\"file\", \"sentence_id\", \"tokenized_text\", \"proper_nouns\", \"length\"])\n",
    "\n",
    "\t\t# For Portuguese, let's try another Portuguese model that is on HF, but is not OPUS-MT\n",
    "\t\tif lang_code == \"PT\":\n",
    "\t\t\tnew_translation_list = []\n",
    "\n",
    "\t\t\tfrom transformers import MBart50TokenizerFast, MBartForConditionalGeneration\n",
    "\t\t\tckpt = 'Narrativa/mbart-large-50-finetuned-opus-pt-en-translation'\n",
    "\n",
    "\t\t\ttokenizer = MBart50TokenizerFast.from_pretrained(ckpt)\n",
    "\t\t\tmodel = MBartForConditionalGeneration.from_pretrained(ckpt).to(\"cuda\")\n",
    "\n",
    "\t\t\ttokenizer.src_lang = 'pt_XX'\n",
    "\n",
    "\t\t\tdef translate(text):\n",
    "\t\t\t\tinputs = tokenizer(text, return_tensors='pt')\n",
    "\t\t\t\tinput_ids = inputs.input_ids.to('cuda')\n",
    "\t\t\t\tattention_mask = inputs.attention_mask.to('cuda')\n",
    "\t\t\t\toutput = model.generate(input_ids, attention_mask=attention_mask, forced_bos_token_id=tokenizer.lang_code_to_id['en_XX'])\n",
    "\t\t\t\treturn tokenizer.decode(output[0], skip_special_tokens=True)\n",
    "\n",
    "\t\t\tfor sentence in sentence_list:\n",
    "\t\t\t\ttranslation = translate(sentence)\n",
    "\t\t\t\tnew_translation_list.append(translation)\n",
    "\t\t\t\n",
    "\t\t\tdf[\"translation-narrativa\"] = new_translation_list\n",
    "\t\t\n",
    "\t\t# For Greek, let's try another model, that does not work with EasyNMT, but is on HF\n",
    "\t\tif lang_code == \"GR\":\n",
    "\t\t\tfrom transformers import pipeline\n",
    "\t\t\tnew_translation_list = []\n",
    "\t\t\tpipe = pipeline(\"translation\", model=\"Helsinki-NLP/opus-mt-tc-big-el-en\")\n",
    "\t\t\tfor sentence in sentence_list:\n",
    "\t\t\t\ttranslation = pipe(sentence)\n",
    "\t\t\t\tnew_translation_list.append(translation[0][\"translation_text\"])\n",
    "\t\t\tdf[\"translation-tc-big-el-en\"] = new_translation_list\n",
    "\n",
    "\t\t# For Basque, let's try another model that is on HF - NLLB (documentation here: https://huggingface.co/docs/transformers/main/en/model_doc/nllb#transformers.NllbTokenizer)\n",
    "\t\tif lang_code == \"ES-PV-eu\":\n",
    "\t\t\tnew_translation_list = []\n",
    "\n",
    "\t\t\tfrom transformers import AutoModelForSeq2SeqLM, AutoTokenizer\n",
    "\n",
    "\t\t\ttokenizer = AutoTokenizer.from_pretrained(\"facebook/nllb-200-distilled-600M\", src_lang=\"eus_Latn\")\n",
    "\t\t\tmodel = AutoModelForSeq2SeqLM.from_pretrained(\"facebook/nllb-200-distilled-600M\")\n",
    "\n",
    "\t\t\tdef translate(text):\n",
    "\t\t\t\tinputs = tokenizer(text, return_tensors='pt')\n",
    "\t\t\t\toutput = model.generate(**inputs, forced_bos_token_id=tokenizer.lang_code_to_id[\"eng_Latn\"], max_length=300\n",
    "\t\t\t)\n",
    "\t\t\t\treturn tokenizer.decode(output[0], skip_special_tokens=True)\n",
    "\n",
    "\t\t\tfor sentence in sentence_list:\n",
    "\t\t\t\ttranslation = translate(sentence)\n",
    "\t\t\t\tnew_translation_list.append(translation)\n",
    "\t\t\t\n",
    "\t\t\tdf[\"translation-nllb200\"] = new_translation_list\n",
    "\n",
    "\t\t# Save the df\n",
    "\t\tdf.to_csv(\"/home/tajak/Parlamint-translation/results/{}/ParlaMint-{}-sample-model-comparison.csv\".format(lang_code, lang_code))\n",
    "\n",
    "\t\tprint(\"The file is saved as/home/tajak/Parlamint-translation/results/{}/ParlaMint-{}-sample-model-comparison.csv. \".format(lang_code, lang_code))\n",
    "\n",
    "\treturn df\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Entire corpus has 247085 sentences and 2893989 words.\n",
      "Sample files has 30 sentences and 338 words.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/tajak/Parlamint-translation/parlamint_env/lib/python3.8/site-packages/transformers/generation/utils.py:1273: UserWarning: Neither `max_length` nor `max_new_tokens` has been set, `max_length` will default to 512 (`generation_config.max_length`). Controlling `max_length` via the config is deprecated and `max_length` will be removed from the config in v5 of Transformers -- we recommend using `max_new_tokens` to control the maximum length of the generation.\n",
      "  warnings.warn(\n",
      "/home/tajak/Parlamint-translation/parlamint_env/lib/python3.8/site-packages/transformers/generation/utils.py:1273: UserWarning: Neither `max_length` nor `max_new_tokens` has been set, `max_length` will default to 512 (`generation_config.max_length`). Controlling `max_length` via the config is deprecated and `max_length` will be removed from the config in v5 of Transformers -- we recommend using `max_new_tokens` to control the maximum length of the generation.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The file is saved as/home/tajak/Parlamint-translation/results/ES-PV-eu/ParlaMint-ES-PV-eu-sample-model-comparison.csv. \n"
     ]
    }
   ],
   "source": [
    "df = choose_model(lang_code, extracted_dataframe_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>file_path</th>\n",
       "      <th>text</th>\n",
       "      <th>lang</th>\n",
       "      <th>translation-eu</th>\n",
       "      <th>translation-mul</th>\n",
       "      <th>translation-nllb200</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1170</th>\n",
       "      <td>/home/tajak/Parlamint-translation/Source-data/...</td>\n",
       "      <td>Egun on guztioi.</td>\n",
       "      <td>eu</td>\n",
       "      <td>Good morning, everyone.</td>\n",
       "      <td>Good morning.</td>\n",
       "      <td>Good morning to you all.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1171</th>\n",
       "      <td>/home/tajak/Parlamint-translation/Source-data/...</td>\n",
       "      <td>Osoko bilkurari hasiera emango diogu.</td>\n",
       "      <td>eu</td>\n",
       "      <td>We'll start the whole meeting.</td>\n",
       "      <td>We'll give him a start in the whole truck.</td>\n",
       "      <td>We're going to start the whole meeting.</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                              file_path  \\\n",
       "1170  /home/tajak/Parlamint-translation/Source-data/...   \n",
       "1171  /home/tajak/Parlamint-translation/Source-data/...   \n",
       "\n",
       "                                       text lang  \\\n",
       "1170                       Egun on guztioi.   eu   \n",
       "1171  Osoko bilkurari hasiera emango diogu.   eu   \n",
       "\n",
       "                      translation-eu  \\\n",
       "1170         Good morning, everyone.   \n",
       "1171  We'll start the whole meeting.   \n",
       "\n",
       "                                 translation-mul  \\\n",
       "1170                               Good morning.   \n",
       "1171  We'll give him a start in the whole truck.   \n",
       "\n",
       "                          translation-nllb200  \n",
       "1170                 Good morning to you all.  \n",
       "1171  We're going to start the whole meeting.  "
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Then open the sample and manually evaluate which model is better in the column \"comparison\"\n",
    "# Open the analysed sample\n",
    "\n",
    "sample = pd.read_csv(\"/home/tajak/Parlamint-translation/results/{}/ParlaMint-{}-sample-model-comparison.csv\".format(lang_code, lang_code), index_col = 0)\n",
    "sample.head(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "zls    10\n",
       "sla     1\n",
       "Name: comparison, dtype: int64"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sample.comparison.value_counts()"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We will use the \"zls\" model."
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "parlamint_env",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "7e8622831a220209edc4d2d4e58bc2159575ea2f8d9419209393154757ff5f93"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}