Label_Studio_SemRel_Annotation_Processing.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "mount_file_id": "1LiOFFW7pi_Q2gFPygNVjViti8I1EwkVj",
      "authorship_tag": "ABX9TyPKMntjxOheEGh8og3UZB7V",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/shmuhammadd/labelstudio-semrel-pipeline/blob/main/Label_Studio_SemRel_Annotation_Processing.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Testing code"
      ],
      "metadata": {
        "id": "_r4PtaOevM84"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "!git clone https://github.com/shmuhammadd/semantic_relatedness.git\n",
        "%cd semantic_relatedness\n",
        "\n",
        "# upload sentence pairs and pilot hausa annotations for testing\n",
        "%cp /content/drive/MyDrive/SemRel/output_news.tsv data/output_news.tsv\n",
        "%cp /content/drive/MyDrive/SemRel/hausa-pilot-annotations.csv data/hausa-pilot-annotations.csv"
      ],
      "metadata": {
        "id": "9687ZnOK-ttx"
      },
      "execution_count": 4,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# generate 4 pair tuples\n",
        "\n",
        "!perl Best-Worst-Scaling-Scripts/generate-BWS-tuples.pl data/output_news.tsv"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "DPXQM5sw_Nwg",
        "outputId": "6633015b-b85e-4388-e758-d0e91b2e5c51"
      },
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Reading data/output_news.tsv ... \n",
            "WARNING: duplicate item (Idan ba a manta ba, kwanan baya ‘yan bindiga sun kashe sojoji 30 a wurin haƙar ma’adinai a Jihar Neja.\tMahara sun kashe sojoji 30, mobal 7, farar hula da dama, kuma sun arce da ‘yan Chana a wurin haƙar ma’adinai da ke Shiroro.); will be included only once.\n",
            "Read 3201 unique items.\n",
            "\n",
            "Generating 6402 4-tuples ...\n",
            "Running 100 iterations ...\n",
            "iteration 1\n",
            "iteration 2\n",
            "iteration 3\n",
            "iteration 4\n",
            "iteration 5\n",
            "iteration 6\n",
            "iteration 7\n",
            "iteration 8\n",
            "iteration 9\n",
            "iteration 10\n",
            "iteration 11\n",
            "iteration 12\n",
            "iteration 13\n",
            "iteration 14\n",
            "iteration 15\n",
            "iteration 16\n",
            "iteration 17\n",
            "iteration 18\n",
            "iteration 19\n",
            "iteration 20\n",
            "iteration 21\n",
            "iteration 22\n",
            "iteration 23\n",
            "iteration 24\n",
            "iteration 25\n",
            "iteration 26\n",
            "iteration 27\n",
            "iteration 28\n",
            "iteration 29\n",
            "iteration 30\n",
            "iteration 31\n",
            "iteration 32\n",
            "iteration 33\n",
            "iteration 34\n",
            "iteration 35\n",
            "iteration 36\n",
            "iteration 37\n",
            "iteration 38\n",
            "iteration 39\n",
            "iteration 40\n",
            "iteration 41\n",
            "iteration 42\n",
            "iteration 43\n",
            "iteration 44\n",
            "iteration 45\n",
            "iteration 46\n",
            "iteration 47\n",
            "iteration 48\n",
            "iteration 49\n",
            "iteration 50\n",
            "iteration 51\n",
            "iteration 52\n",
            "iteration 53\n",
            "iteration 54\n",
            "iteration 55\n",
            "iteration 56\n",
            "iteration 57\n",
            "iteration 58\n",
            "iteration 59\n",
            "iteration 60\n",
            "iteration 61\n",
            "iteration 62\n",
            "iteration 63\n",
            "iteration 64\n",
            "iteration 65\n",
            "iteration 66\n",
            "iteration 67\n",
            "iteration 68\n",
            "iteration 69\n",
            "iteration 70\n",
            "iteration 71\n",
            "iteration 72\n",
            "iteration 73\n",
            "iteration 74\n",
            "iteration 75\n",
            "iteration 76\n",
            "iteration 77\n",
            "iteration 78\n",
            "iteration 79\n",
            "iteration 80\n",
            "iteration 81\n",
            "iteration 82\n",
            "iteration 83\n",
            "iteration 84\n",
            "iteration 85\n",
            "iteration 86\n",
            "iteration 87\n",
            "iteration 88\n",
            "iteration 89\n",
            "iteration 90\n",
            "iteration 91\n",
            "iteration 92\n",
            "iteration 93\n",
            "iteration 94\n",
            "iteration 95\n",
            "iteration 96\n",
            "iteration 97\n",
            "iteration 98\n",
            "iteration 99\n",
            "iteration 100\n",
            "\n",
            "Writing the best set of tuples to data/output_news.tsv.tuples ...\n",
            "Finished.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# use the generated tuples to create label studio annotation file\n",
        "\n",
        "!python scripts/label_studio_annotation_format.py -i data/output_news.tsv.tuples -o data"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "8v4X7RKa8rQM",
        "outputId": "ab67b0c1-8e9f-47b9-835d-036479a129f6"
      },
      "execution_count": 33,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "samples saved to: data/label_studio_annotation_samples.tsv\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# processs the exported annotations for evaluation\n",
        "\n",
        "!python scripts/process_annotations.py -i data/hausa-pilot-annotations.csv -o data"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "MTtaG7fGxB1z",
        "outputId": "e99c20c8-ea42-4e53-bade-4d9c518df448"
      },
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "saved processed files to: data/annotation_to_eval.csv and data/id_to_item.csv\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# install evaluation dependency: Text::CSV\n",
        "\n",
        "!apt-get install libtext-csv-perl"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "vWniaG9zuju8",
        "outputId": "1815f1ff-00f9-4e48-a2d3-b77d1006827d"
      },
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Reading package lists... Done\n",
            "Building dependency tree... Done\n",
            "Reading state information... Done\n",
            "The following additional packages will be installed:\n",
            "  libencode-perl libtext-csv-xs-perl\n",
            "The following NEW packages will be installed:\n",
            "  libencode-perl libtext-csv-perl libtext-csv-xs-perl\n",
            "0 upgraded, 3 newly installed, 0 to remove and 15 not upgraded.\n",
            "Need to get 2,033 kB of archives.\n",
            "After this operation, 10.9 MB of additional disk space will be used.\n",
            "Get:1 http://archive.ubuntu.com/ubuntu jammy/universe amd64 libtext-csv-perl all 2.01-1 [108 kB]\n",
            "Get:2 http://archive.ubuntu.com/ubuntu jammy/universe amd64 libencode-perl amd64 3.16-1build1 [1,810 kB]\n",
            "Get:3 http://archive.ubuntu.com/ubuntu jammy/universe amd64 libtext-csv-xs-perl amd64 1.47-1build1 [115 kB]\n",
            "Fetched 2,033 kB in 0s (5,632 kB/s)\n",
            "Selecting previously unselected package libtext-csv-perl.\n",
            "(Reading database ... 120500 files and directories currently installed.)\n",
            "Preparing to unpack .../libtext-csv-perl_2.01-1_all.deb ...\n",
            "Unpacking libtext-csv-perl (2.01-1) ...\n",
            "Selecting previously unselected package libencode-perl:amd64.\n",
            "Preparing to unpack .../libencode-perl_3.16-1build1_amd64.deb ...\n",
            "Unpacking libencode-perl:amd64 (3.16-1build1) ...\n",
            "Selecting previously unselected package libtext-csv-xs-perl:amd64.\n",
            "Preparing to unpack .../libtext-csv-xs-perl_1.47-1build1_amd64.deb ...\n",
            "Unpacking libtext-csv-xs-perl:amd64 (1.47-1build1) ...\n",
            "Setting up libencode-perl:amd64 (3.16-1build1) ...\n",
            "Setting up libtext-csv-perl (2.01-1) ...\n",
            "Setting up libtext-csv-xs-perl:amd64 (1.47-1build1) ...\n",
            "Processing triggers for man-db (2.10.2-1) ...\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# generate the semantic relatedness scores\n",
        "!perl Best-Worst-Scaling-Scripts/get-scores-from-BWS-annotations-counting.pl \\\n",
        "  data/annotation_to_eval.csv data/pair-scores.tsv\n",
        "\n",
        "# create the semantic relatedness sentence-score pairs\n",
        "!python scripts/create_pair_and_scores.py -i data/id_to_item.csv -s data/pair-scores.tsv -o data"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Aln8jsjgyA2Z",
        "outputId": "a45314f7-107b-4800-97ec-f90f5f72b587"
      },
      "execution_count": 11,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Reading the annotation file data/annotation_to_eval.csv ...\n",
            "Read 405 annotations.\n",
            "Found 416 unique items.\n",
            "\n",
            "Writing the best set of tuples to data/pair-scores.tsv ...\n",
            "Finished.\n"
          ]
        }
      ]
    }
  ]
}