chapter_09/chapter_09.ipynb

{
  "cells": [
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {
        "id": "_vdpDyXVSy4S"
      },
      "outputs": [],
      "source": [
        "!python3 -m pip install spacy==3.7.4"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "s2A2QzvFTOMz"
      },
      "outputs": [],
      "source": [
        "!python -m pip install weasel==0.4.1"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 5,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "lnG9PJsQTEMf",
        "outputId": "ef602df8-8749-4261-b658-d41dceaa5197"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\u001b[38;5;2m✔ Cloned 'tutorials/textcat_goemotions' from 'explosion/projects'\n",
            "(branch 'v3')\u001b[0m\n",
            "/content/textcat_goemotions\n",
            "\u001b[38;5;2m✔ Your project is now ready!\u001b[0m\n",
            "To fetch the assets, run:\n",
            "python -m weasel assets /content/textcat_goemotions\n"
          ]
        }
      ],
      "source": [
        "!python -m weasel clone tutorials/textcat_goemotions"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 6,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "H3NA_MwCT0Xk",
        "outputId": "6f013190-6a68-476b-ebcd-61b786590ebf"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\u001b[38;5;4mℹ Fetching 4 asset(s)\u001b[0m\n",
            "\u001b[38;5;2m✔ Downloaded asset\n",
            "/content/textcat_goemotions/assets/categories.txt\u001b[0m\n",
            "\u001b[38;5;2m✔ Downloaded asset /content/textcat_goemotions/assets/train.tsv\u001b[0m\n",
            "\u001b[38;5;2m✔ Downloaded asset /content/textcat_goemotions/assets/dev.tsv\u001b[0m\n",
            "\u001b[38;5;2m✔ Downloaded asset /content/textcat_goemotions/assets/test.tsv\u001b[0m\n"
          ]
        }
      ],
      "source": [
        "!cd ./textcat_goemotions/ && python3 -m weasel assets"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "LpNVBZyXUEel",
        "outputId": "4492682c-88be-439a-e86c-3ac34c8874f6"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\u001b[38;5;4mℹ Running workflow 'all'\u001b[0m\n",
            "\u001b[1m\n",
            "================================= preprocess =================================\u001b[0m\n",
            "Running command: /usr/bin/python3 scripts/convert_corpus.py\n",
            "\u001b[1m\n",
            "=================================== train ===================================\u001b[0m\n",
            "Running command: /usr/bin/python3 -m spacy train ./configs/cnn.cfg -o training/cnn --gpu-id -1\n",
            "\u001b[38;5;2m✔ Created output directory: training/cnn\u001b[0m\n",
            "\u001b[38;5;4mℹ Saving to output directory: training/cnn\u001b[0m\n",
            "\u001b[38;5;4mℹ Using CPU\u001b[0m\n",
            "\u001b[38;5;4mℹ To switch to GPU 0, use the option: --gpu-id 0\u001b[0m\n",
            "\u001b[1m\n",
            "=========================== Initializing pipeline ===========================\u001b[0m\n",
            "\u001b[38;5;2m✔ Initialized pipeline\u001b[0m\n",
            "\u001b[1m\n",
            "============================= Training pipeline =============================\u001b[0m\n",
            "\u001b[38;5;4mℹ Pipeline: ['textcat']\u001b[0m\n",
            "\u001b[38;5;4mℹ Initial learn rate: 0.001\u001b[0m\n",
            "E    #       LOSS TEXTCAT  CATS_SCORE  SCORE \n",
            "---  ------  ------------  ----------  ------\n",
            "  0       0          0.27       50.46    0.50\n",
            "  0     200          7.96       57.03    0.57\n",
            "  0     400          7.04       60.02    0.60\n",
            "  0     600          6.61       62.25    0.62\n",
            "  0     800          6.45       63.64    0.64\n",
            "  0    1000          6.34       66.80    0.67\n",
            "  0    1200          6.19       69.88    0.70\n",
            "  1    1400          6.02       72.13    0.72\n",
            "  1    1600          5.67       73.92    0.74\n",
            "  1    1800          5.74       75.33    0.75\n",
            "  1    2000          5.80       76.51    0.77\n",
            "  1    2200          5.65       76.74    0.77\n",
            "  1    2400          5.60       77.74    0.78\n",
            "  1    2600          5.63       78.36    0.78\n",
            "  2    2800          5.41       78.72    0.79\n",
            "  2    3000          5.30       79.01    0.79\n",
            "  2    3200          5.33       79.43    0.79\n",
            "  2    3400          5.23       79.99    0.80\n",
            "  2    3600          5.32       80.21    0.80\n",
            "  2    3800          5.19       80.39    0.80\n",
            "  2    4000          5.34       80.51    0.81\n",
            "  3    4200          5.00       80.80    0.81\n",
            "  3    4400          4.93       80.98    0.81\n",
            "  3    4600          4.99       81.28    0.81\n",
            "  3    4800          4.90       81.53    0.82\n",
            "  3    5000          5.00       81.84    0.82\n",
            "  3    5200          5.02       82.14    0.82\n",
            "  3    5400          5.07       82.23    0.82\n",
            "  4    5600          4.69       82.31    0.82\n",
            "  4    5800          4.68       82.43    0.82\n",
            "  4    6000          4.77       82.53    0.83\n",
            "  4    6200          4.63       82.72    0.83\n",
            "  4    6400          4.85       82.84    0.83\n",
            "  4    6600          4.77       82.92    0.83\n",
            "  5    6800          4.65       83.11    0.83\n",
            "  5    7000          4.38       83.25    0.83\n",
            "  5    7200          4.39       83.41    0.83\n",
            "  5    7400          4.42       83.48    0.83\n",
            "  5    7600          4.53       83.58    0.84\n",
            "  5    7800          4.59       83.61    0.84\n",
            "  5    8000          4.47       83.61    0.84\n",
            "  6    8200          4.46       83.68    0.84\n",
            "  6    8400          4.08       83.75    0.84\n",
            "  6    8600          4.36       83.73    0.84\n"
          ]
        }
      ],
      "source": [
        "!cd ./textcat_goemotions/ && python -m weasel run all"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 3,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "hdBwLfa5NkYx",
        "outputId": "1ea9e523-170d-4eda-ca0c-69848deb6af5"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\u001b[38;5;2m✔ Cloned 'tutorials/textcat_goemotions' from 'explosion/projects'\n",
            "(branch 'v3')\u001b[0m\n",
            "/content/textcat_github_issues\n",
            "\u001b[38;5;2m✔ Your project is now ready!\u001b[0m\n",
            "To fetch the assets, run:\n",
            "python -m weasel assets textcat_github_issues\n"
          ]
        }
      ],
      "source": [
        "!python -m weasel clone tutorials/textcat_goemotions textcat_github_issues"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 4,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "2Zuj6vu-Nw52",
        "outputId": "64fb21c4-cf56-41a4-878e-fae9540010b8"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\u001b[38;5;4mℹ Fetching 3 asset(s)\u001b[0m\n",
            "\u001b[38;5;2m✔ Downloaded asset\n",
            "/content/textcat_github_issues/assets/train.jsonl\u001b[0m\n",
            "\u001b[38;5;2m✔ Downloaded asset /content/textcat_github_issues/assets/dev.jsonl\u001b[0m\n",
            "\u001b[38;5;2m✔ Downloaded asset /content/textcat_github_issues/assets/eval.jsonl\u001b[0m\n"
          ]
        }
      ],
      "source": [
        "!cd ./textcat_github_issues/ && python3 -m weasel assets"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 5,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "-BMtymjFOhrv",
        "outputId": "2115c424-5281-42b0-ca18-78c5a18c7b71"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "\u001b[38;5;1m✘ Missing dependency specified by command 'preprocess':\n",
            "assets/train.tsv\u001b[0m\n",
            "Maybe you forgot to run the 'project assets' command or a previous step?\n",
            "\n"
          ]
        }
      ],
      "source": [
        "!cd ./textcat_github_issues/ && python -m weasel run preprocess"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 7,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ZM-Y5iy9RLbG",
        "outputId": "573303f1-1bbe-412f-a93c-3157253eb24c"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\u001b[1m\n",
            "================================= preprocess =================================\u001b[0m\n",
            "Running command: /usr/bin/python3 scripts/convert_corpus.py\n"
          ]
        }
      ],
      "source": [
        "!cd ./textcat_github_issues/ && python -m weasel run preprocess"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 12,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "z9uEiMmVSCmK",
        "outputId": "69b812ea-bdf5-4941-95ef-270d7c0b7f44"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\u001b[38;5;4mℹ Running workflow 'all'\u001b[0m\n",
            "\u001b[1m\n",
            "================================= preprocess =================================\u001b[0m\n",
            "\u001b[38;5;4mℹ Skipping 'preprocess': nothing changed\u001b[0m\n",
            "\u001b[1m\n",
            "=================================== train ===================================\u001b[0m\n",
            "Running command: /usr/bin/python3 -m spacy train ./configs/cnn.cfg -o training/cnn --gpu-id 0\n",
            "\u001b[38;5;4mℹ Saving to output directory: training/cnn\u001b[0m\n",
            "\u001b[38;5;4mℹ Using GPU: 0\u001b[0m\n",
            "\u001b[1m\n",
            "=========================== Initializing pipeline ===========================\u001b[0m\n",
            "\u001b[38;5;2m✔ Initialized pipeline\u001b[0m\n",
            "\u001b[1m\n",
            "============================= Training pipeline =============================\u001b[0m\n",
            "\u001b[38;5;4mℹ Pipeline: ['textcat']\u001b[0m\n",
            "\u001b[38;5;4mℹ Initial learn rate: 0.001\u001b[0m\n",
            "E    #       LOSS TEXTCAT  CATS_SCORE  SCORE \n",
            "---  ------  ------------  ----------  ------\n",
            "  0       0          0.31       46.49    0.46\n",
            "  9     200          9.85       85.30    0.85\n",
            " 19     400          0.16       84.46    0.84\n",
            " 28     600          0.06       84.03    0.84\n",
            " 38     800          0.03       84.49    0.84\n",
            " 47    1000          0.05       84.50    0.84\n",
            " 57    1200          0.02       84.13    0.84\n",
            " 66    1400          0.00       84.24    0.84\n",
            " 76    1600          0.03       83.95    0.84\n",
            " 85    1800          0.02       83.40    0.83\n",
            "\u001b[38;5;2m✔ Saved pipeline to output directory\u001b[0m\n",
            "training/cnn/model-last\n",
            "\n",
            "\u001b[38;5;1m✘ Missing dependency specified by command 'evaluate':\n",
            "corpus/test.spacy\u001b[0m\n",
            "Maybe you forgot to run the 'project assets' command or a previous step?\n",
            "\n"
          ]
        }
      ],
      "source": [
        "!cd ./textcat_github_issues/ && python -m weasel run all"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 15,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ymwWmmmDXCDS",
        "outputId": "3dcc6340-abc2-43f0-ef34-1efecf3d314f"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\u001b[1m\n",
            "================================== evaluate ==================================\u001b[0m\n",
            "Running command: /usr/bin/python3 -m spacy evaluate ./training/cnn/model-best ./corpus/eval.spacy --output ./metrics/cnn.json\n",
            "\u001b[38;5;4mℹ Using CPU\u001b[0m\n",
            "\u001b[38;5;4mℹ To switch to GPU 0, use the option: --gpu-id 0\u001b[0m\n",
            "\u001b[1m\n",
            "================================== Results ==================================\u001b[0m\n",
            "\n",
            "TOK                   100.00\n",
            "TEXTCAT (macro AUC)   77.88 \n",
            "SPEED                 19801 \n",
            "\n",
            "\u001b[1m\n",
            "=========================== Textcat F (per label) ===========================\u001b[0m\n",
            "\n",
            "                    P       R       F\n",
            "DOCUMENTATION   62.94   74.31   68.15\n",
            "OTHER           71.97   60.90   65.97\n",
            "\n",
            "\u001b[1m\n",
            "======================== Textcat ROC AUC (per label) ========================\u001b[0m\n",
            "\n",
            "                ROC AUC\n",
            "DOCUMENTATION      0.78\n",
            "OTHER              0.78\n",
            "\n",
            "\u001b[38;5;2m✔ Saved results to metrics/cnn.json\u001b[0m\n"
          ]
        }
      ],
      "source": [
        "!cd ./textcat_github_issues/ && python -m weasel run evaluate"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "oZjukNS9Xr2X"
      },
      "outputs": [],
      "source": [
        "!python -m pip install spacy-transformers==1.3.5"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "f6QK3AK5X4o2"
      },
      "outputs": [],
      "source": [
        "!python -m spacy download en_core_web_lg"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "jSYtXXruXk65",
        "outputId": "190b0904-ea1e-4a74-f17c-c44d38169e6d"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\u001b[38;5;4mℹ Running workflow 'all'\u001b[0m\n",
            "\u001b[1m\n",
            "================================= preprocess =================================\u001b[0m\n",
            "\u001b[38;5;4mℹ Skipping 'preprocess': nothing changed\u001b[0m\n",
            "\u001b[1m\n",
            "=================================== train ===================================\u001b[0m\n",
            "Running command: /usr/bin/python3 -m spacy train ./configs/bert.cfg -o training/bert --gpu-id 0\n",
            "\u001b[38;5;4mℹ Saving to output directory: training/bert\u001b[0m\n",
            "\u001b[38;5;4mℹ Using GPU: 0\u001b[0m\n",
            "\u001b[1m\n",
            "=========================== Initializing pipeline ===========================\u001b[0m\n",
            "/usr/local/lib/python3.10/dist-packages/transformers/utils/generic.py:441: UserWarning: torch.utils._pytree._register_pytree_node is deprecated. Please use torch.utils._pytree.register_pytree_node instead.\n",
            "  _torch_pytree._register_pytree_node(\n",
            "/usr/local/lib/python3.10/dist-packages/transformers/utils/generic.py:309: UserWarning: torch.utils._pytree._register_pytree_node is deprecated. Please use torch.utils._pytree.register_pytree_node instead.\n",
            "  _torch_pytree._register_pytree_node(\n",
            "/usr/local/lib/python3.10/dist-packages/huggingface_hub/file_download.py:1132: FutureWarning: `resume_download` is deprecated and will be removed in version 1.0.0. Downloads always resume when possible. If you want to force a new download, use `force_download=True`.\n",
            "  warnings.warn(\n",
            "tokenizer_config.json: 100% 25.0/25.0 [00:00<00:00, 138kB/s]\n",
            "config.json: 100% 481/481 [00:00<00:00, 2.45MB/s]\n",
            "vocab.json: 100% 899k/899k [00:00<00:00, 4.09MB/s]\n",
            "merges.txt: 100% 456k/456k [00:00<00:00, 2.12MB/s]\n",
            "tokenizer.json: 100% 1.36M/1.36M [00:00<00:00, 3.15MB/s]\n",
            "/usr/local/lib/python3.10/dist-packages/transformers/utils/generic.py:309: UserWarning: torch.utils._pytree._register_pytree_node is deprecated. Please use torch.utils._pytree.register_pytree_node instead.\n",
            "  _torch_pytree._register_pytree_node(\n",
            "/usr/local/lib/python3.10/dist-packages/huggingface_hub/file_download.py:1132: FutureWarning: `resume_download` is deprecated and will be removed in version 1.0.0. Downloads always resume when possible. If you want to force a new download, use `force_download=True`.\n",
            "  warnings.warn(\n",
            "model.safetensors: 100% 499M/499M [00:01<00:00, 301MB/s]\n",
            "Some weights of RobertaModel were not initialized from the model checkpoint at roberta-base and are newly initialized: ['roberta.pooler.dense.weight', 'roberta.pooler.dense.bias']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",
            "\u001b[38;5;2m✔ Initialized pipeline\u001b[0m\n",
            "\u001b[1m\n",
            "============================= Training pipeline =============================\u001b[0m\n",
            "\u001b[38;5;4mℹ Pipeline: ['transformer', 'textcat']\u001b[0m\n",
            "\u001b[38;5;4mℹ Initial learn rate: 0.0\u001b[0m\n",
            "E    #       LOSS TRANS...  LOSS TEXTCAT  CATS_SCORE  CATS_MACRO_F  SCORE \n",
            "---  ------  -------------  ------------  ----------  ------------  ------\n",
            "Token indices sequence length is longer than the specified maximum sequence length for this model (1087 > 512). Running this sequence through the model will result in indexing errors\n",
            "  0       0           0.00          1.40       51.85          0.60    0.26\n",
            " 18     400          16.53         82.50       93.87         88.70    0.91\n",
            " 37     800           0.37          3.61       95.80         88.78    0.92\n"
          ]
        }
      ],
      "source": [
        "!cd ./textcat_github_issues/ && python -m weasel run all"
      ]
    }
  ],
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "gpuType": "T4",
      "provenance": []
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python",
      "version": "3.10.12"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}