CoNLL-U_parser.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import pandas as pd\n",
    "from time import time"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Parsing the `.conllu` file line by lines\n",
    "`records` be an empty list of lists to hold tagged information"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Time elapsed: 0.07933378219604492 seconds\n"
     ]
    }
   ],
   "source": [
    "start = time()\n",
    "file_path = 'hi_pud-ud-test.conllu' # path where your .conllu file is located\n",
    "with open(file_path, 'r', encoding='utf-8') as file:\n",
    "    file_prefix = file_path.split('.')[0] + '_'\n",
    "    doc_id = ''\n",
    "    sent_id = ''\n",
    "    records = list()\n",
    "    for line in file:\n",
    "        if len(line) > 1 :\n",
    "            if line[0] == '#':\n",
    "                line = line.split('=')\n",
    "                if 'newdoc' in line[0]:\n",
    "                    doc_id = file_prefix + line[1].strip()\n",
    "                elif 'sent_id' in line[0]:\n",
    "                    sent_id = line[1].strip()\n",
    "            else:\n",
    "                info = line.split('\\t')\n",
    "                if len(info) == 10:\n",
    "                    records.append([doc_id, sent_id] + [x.strip() for x in info])\n",
    "end = time()\n",
    "print(\"Time elapsed:\", end-start, \"seconds\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Create a dataframe to copy the parsed records"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.DataFrame(records, columns=['DOC_NO', 'SENT_NO', 'ID', 'FORM','LEMMA','UPOS', 'XPOS', 'FEAT','HEAD', 'DEPREL', 'DEPS', 'MISC'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Saving parsed records in tsv (tab separated view) file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>DOC_NO</th>\n",
       "      <th>SENT_NO</th>\n",
       "      <th>ID</th>\n",
       "      <th>FORM</th>\n",
       "      <th>LEMMA</th>\n",
       "      <th>UPOS</th>\n",
       "      <th>XPOS</th>\n",
       "      <th>FEAT</th>\n",
       "      <th>HEAD</th>\n",
       "      <th>DEPREL</th>\n",
       "      <th>DEPS</th>\n",
       "      <th>MISC</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>hi_pud-ud-test_n01001</td>\n",
       "      <td>n01001011</td>\n",
       "      <td>1</td>\n",
       "      <td>“</td>\n",
       "      <td>_</td>\n",
       "      <td>PUNCT</td>\n",
       "      <td>``</td>\n",
       "      <td>_</td>\n",
       "      <td>16</td>\n",
       "      <td>punct</td>\n",
       "      <td>_</td>\n",
       "      <td>SpaceAfter=No|Translit=“</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>hi_pud-ud-test_n01001</td>\n",
       "      <td>n01001011</td>\n",
       "      <td>2</td>\n",
       "      <td>संयुक्त</td>\n",
       "      <td>_</td>\n",
       "      <td>ADJ</td>\n",
       "      <td>JJ</td>\n",
       "      <td>_</td>\n",
       "      <td>4</td>\n",
       "      <td>amod</td>\n",
       "      <td>_</td>\n",
       "      <td>Proper=True|Translit=saṁyukta</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>hi_pud-ud-test_n01001</td>\n",
       "      <td>n01001011</td>\n",
       "      <td>3</td>\n",
       "      <td>राज्य</td>\n",
       "      <td>_</td>\n",
       "      <td>NOUN</td>\n",
       "      <td>NN</td>\n",
       "      <td>Animacy=Inan|Gender=Masc|Number=Sing</td>\n",
       "      <td>4</td>\n",
       "      <td>compound</td>\n",
       "      <td>_</td>\n",
       "      <td>Proper=True|Translit=rājya</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>hi_pud-ud-test_n01001</td>\n",
       "      <td>n01001011</td>\n",
       "      <td>4</td>\n",
       "      <td>अमेरिका</td>\n",
       "      <td>_</td>\n",
       "      <td>PROPN</td>\n",
       "      <td>NNP</td>\n",
       "      <td>Animacy=Inan|Case=Acc|Gender=Masc|Number=Sing</td>\n",
       "      <td>16</td>\n",
       "      <td>obl</td>\n",
       "      <td>_</td>\n",
       "      <td>Translit=amerikā</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>hi_pud-ud-test_n01001</td>\n",
       "      <td>n01001011</td>\n",
       "      <td>5</td>\n",
       "      <td>में</td>\n",
       "      <td>_</td>\n",
       "      <td>ADP</td>\n",
       "      <td>IN</td>\n",
       "      <td>Case=Loc</td>\n",
       "      <td>4</td>\n",
       "      <td>case</td>\n",
       "      <td>_</td>\n",
       "      <td>Translit=meṁ</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                  DOC_NO    SENT_NO ID     FORM LEMMA   UPOS XPOS  \\\n",
       "0  hi_pud-ud-test_n01001  n01001011  1        “     _  PUNCT   ``   \n",
       "1  hi_pud-ud-test_n01001  n01001011  2  संयुक्त     _    ADJ   JJ   \n",
       "2  hi_pud-ud-test_n01001  n01001011  3    राज्य     _   NOUN   NN   \n",
       "3  hi_pud-ud-test_n01001  n01001011  4  अमेरिका     _  PROPN  NNP   \n",
       "4  hi_pud-ud-test_n01001  n01001011  5      में     _    ADP   IN   \n",
       "\n",
       "                                            FEAT HEAD    DEPREL DEPS  \\\n",
       "0                                              _   16     punct    _   \n",
       "1                                              _    4      amod    _   \n",
       "2           Animacy=Inan|Gender=Masc|Number=Sing    4  compound    _   \n",
       "3  Animacy=Inan|Case=Acc|Gender=Masc|Number=Sing   16       obl    _   \n",
       "4                                       Case=Loc    4      case    _   \n",
       "\n",
       "                            MISC  \n",
       "0       SpaceAfter=No|Translit=“  \n",
       "1  Proper=True|Translit=saṁyukta  \n",
       "2     Proper=True|Translit=rājya  \n",
       "3               Translit=amerikā  \n",
       "4                   Translit=meṁ  "
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.to_csv('hindi-output.tsv', header=True, index=None, sep='\\t')\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Iterating over a folder of .coNLLu files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "aq-set-10.conllu Time elapsed: 0.12796545028686523 seconds\n",
      "aq-set-13.conllu Time elapsed: 0.08791232109069824 seconds\n",
      "aq-set-4.conllu Time elapsed: 0.04593944549560547 seconds\n",
      "aq-set-14.conllu Time elapsed: 0.6223409175872803 seconds\n",
      "aq-set-7.conllu Time elapsed: 0.07025718688964844 seconds\n",
      "aq-set-6.conllu Time elapsed: 0.07943248748779297 seconds\n",
      "aq-set-2.conllu Time elapsed: 0.010993719100952148 seconds\n",
      "aq-set-8.conllu Time elapsed: 0.43564367294311523 seconds\n",
      "aq-set-9.conllu Time elapsed: 0.07404804229736328 seconds\n",
      "aq-set-12.conllu Time elapsed: 0.08122825622558594 seconds\n",
      "aq-set-3.conllu Time elapsed: 0.015195369720458984 seconds\n",
      "aq-set-1.conllu Time elapsed: 0.006630897521972656 seconds\n",
      "aq-set-11.conllu Time elapsed: 0.07516884803771973 seconds\n",
      "aq-set-5.conllu Time elapsed: 0.07778739929199219 seconds\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>DOC_ID</th>\n",
       "      <th>PAR_ID</th>\n",
       "      <th>SENT_ID</th>\n",
       "      <th>ID</th>\n",
       "      <th>FORM</th>\n",
       "      <th>LEMMA</th>\n",
       "      <th>UPOS</th>\n",
       "      <th>XPOS</th>\n",
       "      <th>FEAT</th>\n",
       "      <th>HEAD</th>\n",
       "      <th>DEPREL</th>\n",
       "      <th>DEPS</th>\n",
       "      <th>MISC</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>aq-set-10_1</td>\n",
       "      <td>aq-set-10_1</td>\n",
       "      <td>aq-set-10_1</td>\n",
       "      <td>1</td>\n",
       "      <td>The</td>\n",
       "      <td>the</td>\n",
       "      <td>DET</td>\n",
       "      <td>DT</td>\n",
       "      <td>Definite=Def|PronType=Art</td>\n",
       "      <td>2</td>\n",
       "      <td>det</td>\n",
       "      <td>_</td>\n",
       "      <td>_</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>aq-set-10_1</td>\n",
       "      <td>aq-set-10_1</td>\n",
       "      <td>aq-set-10_1</td>\n",
       "      <td>2</td>\n",
       "      <td>development</td>\n",
       "      <td>development</td>\n",
       "      <td>NOUN</td>\n",
       "      <td>NN</td>\n",
       "      <td>Number=Sing</td>\n",
       "      <td>13</td>\n",
       "      <td>nsubj</td>\n",
       "      <td>_</td>\n",
       "      <td>_</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>aq-set-10_1</td>\n",
       "      <td>aq-set-10_1</td>\n",
       "      <td>aq-set-10_1</td>\n",
       "      <td>3</td>\n",
       "      <td>of</td>\n",
       "      <td>of</td>\n",
       "      <td>ADP</td>\n",
       "      <td>IN</td>\n",
       "      <td>_</td>\n",
       "      <td>4</td>\n",
       "      <td>case</td>\n",
       "      <td>_</td>\n",
       "      <td>_</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>aq-set-10_1</td>\n",
       "      <td>aq-set-10_1</td>\n",
       "      <td>aq-set-10_1</td>\n",
       "      <td>4</td>\n",
       "      <td>telephony</td>\n",
       "      <td>telephony</td>\n",
       "      <td>NOUN</td>\n",
       "      <td>NN</td>\n",
       "      <td>Number=Sing</td>\n",
       "      <td>2</td>\n",
       "      <td>nmod</td>\n",
       "      <td>_</td>\n",
       "      <td>_</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>aq-set-10_1</td>\n",
       "      <td>aq-set-10_1</td>\n",
       "      <td>aq-set-10_1</td>\n",
       "      <td>5</td>\n",
       "      <td>in</td>\n",
       "      <td>in</td>\n",
       "      <td>ADP</td>\n",
       "      <td>IN</td>\n",
       "      <td>_</td>\n",
       "      <td>8</td>\n",
       "      <td>case</td>\n",
       "      <td>_</td>\n",
       "      <td>_</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        DOC_ID       PAR_ID      SENT_ID ID         FORM        LEMMA  UPOS  \\\n",
       "0  aq-set-10_1  aq-set-10_1  aq-set-10_1  1          The          the   DET   \n",
       "1  aq-set-10_1  aq-set-10_1  aq-set-10_1  2  development  development  NOUN   \n",
       "2  aq-set-10_1  aq-set-10_1  aq-set-10_1  3           of           of   ADP   \n",
       "3  aq-set-10_1  aq-set-10_1  aq-set-10_1  4    telephony    telephony  NOUN   \n",
       "4  aq-set-10_1  aq-set-10_1  aq-set-10_1  5           in           in   ADP   \n",
       "\n",
       "  XPOS                       FEAT HEAD DEPREL DEPS MISC  \n",
       "0   DT  Definite=Def|PronType=Art    2    det    _    _  \n",
       "1   NN                Number=Sing   13  nsubj    _    _  \n",
       "2   IN                          _    4   case    _    _  \n",
       "3   NN                Number=Sing    2   nmod    _    _  \n",
       "4   IN                          _    8   case    _    _  "
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "directory = '../gdex-word-embeddings/ud/'\n",
    "records = list()\n",
    "for filename in os.listdir(directory):\n",
    "    if '-set-' in filename and filename.endswith(\".conllu\"):\n",
    "        start = time()\n",
    "        file_path = os.path.join(directory, filename) # path where your .conllu file is located\n",
    "        #print(file_path)\n",
    "        with open(file_path, 'r') as file:\n",
    "            file_prefix = filename.split('.')[0] + '_'\n",
    "            doc_count = 1\n",
    "            sent_count = 1\n",
    "            par_count = 1\n",
    "            doc_id = file_prefix + str(doc_count)\n",
    "            sent_id = sent_count\n",
    "            par_id = par_count            \n",
    "            for line in file:\n",
    "                if len(line) > 1 :\n",
    "                    # if it is related to doc, paragraph or sentence \n",
    "                    if line[0] == '#':\n",
    "                        if 'newdoc' in line:\n",
    "                            if line.strip() != '# newdoc':\n",
    "                                if '=' in line:\n",
    "                                    doc_id = file_prefix + line.split('=')[1].strip()\n",
    "                                    doc_count += 1\n",
    "                                else:\n",
    "                                    doc_id = file_prefix + str(doc_count)\n",
    "                                    doc_count =+ 1\n",
    "                            elif line.strip() == '# newdoc':\n",
    "                                doc_id = file_prefix + str(doc_count)\n",
    "                                doc_count =+ 1\n",
    "                                \n",
    "                        if 'newpar' in line:\n",
    "                            if line.strip() != '# newpar':\n",
    "                                if '=' in line:\n",
    "                                    par_id = file_prefix + line.split('=')[1].strip()\n",
    "                                    par_count += 1\n",
    "                                else:\n",
    "                                    par_id = file_prefix + str(par_count)\n",
    "                                    par_count =+ 1\n",
    "                            elif line.strip() == '# newpar':\n",
    "                                par_id = file_prefix + str(par_count)\n",
    "                                par_count =+ 1\n",
    "                        \n",
    "                        if '# sent_id' in line:\n",
    "                            if '=' in line:\n",
    "                                sent_id = file_prefix + line.split('=')[1].strip()\n",
    "                                sent_count += 1\n",
    "                            else:\n",
    "                                sent_id = file_prefix + str(sent_count)\n",
    "                                sent_count =+ 1\n",
    "                    # if it is related to coNLLu data\n",
    "                    elif line[0] != '#':\n",
    "                        info = line.split('\\t')\n",
    "                        if len(info) == 10:\n",
    "                            temp = [doc_id, par_id, sent_id] + [x.strip() for x in info]\n",
    "                            if len(temp) == 13:\n",
    "                                records.append(temp)\n",
    "                else:\n",
    "                    doc_count += 1\n",
    "                    par_count += 1\n",
    "                    sent_count += 1\n",
    "                    \n",
    "        end = time()\n",
    "        print(filename, \"Time elapsed:\", end-start, \"seconds\")\n",
    "    else:\n",
    "        continue\n",
    "df = pd.DataFrame(records, columns=['DOC_ID', 'PAR_ID', 'SENT_ID', 'ID', 'FORM','LEMMA','UPOS', 'XPOS', 'FEAT','HEAD', 'DEPREL', 'DEPS', 'MISC'])\n",
    "df.to_csv('aq-all-conllu.tsv', header=True, index=None, sep='\\t')\n",
    "df.head()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}