CDLE_NLP_TextClassification_公開.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "collapsed_sections": [
        "63z1zLB5vOsu",
        "h8pnv8-m6CaU",
        "p-OeCLFS54KF",
        "XSMM7a6O6PUz",
        "386HMGIz6SzH",
        "JX1BpfxD6sct",
        "nqo0iaXS6GpG",
        "zA5c3NYM6Xcz",
        "GN_xtOpN6ir2",
        "dytKJUEWai3S",
        "u3JoRd1MapiJ"
      ]
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU",
    "gpuClass": "premium",
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "0815b713b12f43c783e2901a7738b3a4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_1c4bced711c64abd96330f6184673528",
              "IPY_MODEL_965a04241b5d41378b02df443dd528d5",
              "IPY_MODEL_ff4e1c23b9504602968ffb52876cb866"
            ],
            "layout": "IPY_MODEL_e0fa471119b340a7a8d105ec2b127eaf"
          }
        },
        "1c4bced711c64abd96330f6184673528": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_07fd2f385e004657ad0504aa694797fa",
            "placeholder": "​",
            "style": "IPY_MODEL_3147e0cbe84d4c75ab7eb3100b5439d7",
            "value": "Downloading (…)okenizer_config.json: 100%"
          }
        },
        "965a04241b5d41378b02df443dd528d5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_5d948a11af534945ade3aa83dcfa978f",
            "max": 605,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_52b001d10d424c34a683cb8ffd4df813",
            "value": 605
          }
        },
        "ff4e1c23b9504602968ffb52876cb866": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_505e2c55212f41d29a7450d2707a76c0",
            "placeholder": "​",
            "style": "IPY_MODEL_71f17654f080472f96bb6d4a708ad6e5",
            "value": " 605/605 [00:00&lt;00:00, 42.8kB/s]"
          }
        },
        "e0fa471119b340a7a8d105ec2b127eaf": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "07fd2f385e004657ad0504aa694797fa": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "3147e0cbe84d4c75ab7eb3100b5439d7": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "5d948a11af534945ade3aa83dcfa978f": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "52b001d10d424c34a683cb8ffd4df813": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "505e2c55212f41d29a7450d2707a76c0": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "71f17654f080472f96bb6d4a708ad6e5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "ed8bc585f9f74c45880a8641cfd6cb68": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_8cfb40b4837740b7b8b31b23ceda6249",
              "IPY_MODEL_2da72d13e7ba43ddae5dd11a6b8c9beb",
              "IPY_MODEL_0e96c411892b42649e40338ffc0f79b8"
            ],
            "layout": "IPY_MODEL_e9bbd541a00e4cd0b1657dad084f2d9e"
          }
        },
        "8cfb40b4837740b7b8b31b23ceda6249": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_0bab86650a9845cbb5ddf9a2c6c52309",
            "placeholder": "​",
            "style": "IPY_MODEL_48ec0677d3a04a77b3ae1b6e1ef63202",
            "value": "Downloading (…)solve/main/vocab.txt: 100%"
          }
        },
        "2da72d13e7ba43ddae5dd11a6b8c9beb": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_48eb8aebdc1848958e62b7775de17719",
            "max": 236001,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_282c7333409b4d0997386794b668d4a5",
            "value": 236001
          }
        },
        "0e96c411892b42649e40338ffc0f79b8": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_427cd5f36a3d4f31b8133221e0fa39a4",
            "placeholder": "​",
            "style": "IPY_MODEL_ae98d213cd644b188fb20f290ca175be",
            "value": " 236k/236k [00:00&lt;00:00, 562kB/s]"
          }
        },
        "e9bbd541a00e4cd0b1657dad084f2d9e": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "0bab86650a9845cbb5ddf9a2c6c52309": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "48ec0677d3a04a77b3ae1b6e1ef63202": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "48eb8aebdc1848958e62b7775de17719": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "282c7333409b4d0997386794b668d4a5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "427cd5f36a3d4f31b8133221e0fa39a4": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "ae98d213cd644b188fb20f290ca175be": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "9a4e6106b7b34adb8b761adbf7b6d188": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_4fdb1de4e4674fc4802a3b956e62534c",
              "IPY_MODEL_8fb86fbdb86344488181498d3ee4606e",
              "IPY_MODEL_6a93084bcf864d6fac49c592d9b61708"
            ],
            "layout": "IPY_MODEL_00a13910b72746ff815c8c795fd22659"
          }
        },
        "fc795c2a39234dc5872a82722596d6df": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_4bcf544026984b1faf04fc8aa139df5b",
              "IPY_MODEL_db75da41e1d64acb983976356d17b053",
              "IPY_MODEL_5c3f79b9f3a249b4845d6fdc0c6b3074"
            ],
            "layout": "IPY_MODEL_7e1a29c540c04fadbdd531fd0b3340e9"
          }
        },
        "4bcf544026984b1faf04fc8aa139df5b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_f8813ff4d2874fdfa2e98b8d02254350",
            "placeholder": "​",
            "style": "IPY_MODEL_55ba4fa2b283407f9aacdbabeb263dce",
            "value": "Running tokenizer on dataset: 100%"
          }
        },
        "db75da41e1d64acb983976356d17b053": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_aefdb55cc72447968dd259f9e6aebb5a",
            "max": 10000,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_a67e64e771364e3fa4304a3171deb24b",
            "value": 10000
          }
        },
        "5c3f79b9f3a249b4845d6fdc0c6b3074": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_f20d1409c3ee428c84150f4080ff8e79",
            "placeholder": "​",
            "style": "IPY_MODEL_a205277548b64e2eaf1540bf535f7b71",
            "value": " 10000/10000 [00:06&lt;00:00, 1479.74 examples/s]"
          }
        },
        "7e1a29c540c04fadbdd531fd0b3340e9": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": "hidden",
            "width": null
          }
        },
        "f8813ff4d2874fdfa2e98b8d02254350": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "55ba4fa2b283407f9aacdbabeb263dce": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "aefdb55cc72447968dd259f9e6aebb5a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "a67e64e771364e3fa4304a3171deb24b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "f20d1409c3ee428c84150f4080ff8e79": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "a205277548b64e2eaf1540bf535f7b71": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "37ed2d5f107742768279e25a11af9892": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_15daf6f5bfbf453987fdf175eb1dd655",
              "IPY_MODEL_88d1762760f14bfa8271080da19d8e31",
              "IPY_MODEL_3d0bd98d7bf84cee8315de1b32f51342"
            ],
            "layout": "IPY_MODEL_0a036fcd45344c6288d35b8c00a32633"
          }
        },
        "15daf6f5bfbf453987fdf175eb1dd655": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_92a2cc53d1fe4106adcde8e4ce5242e8",
            "placeholder": "​",
            "style": "IPY_MODEL_90360c85b6dd4f098054a89db24c9820",
            "value": "Running tokenizer on dataset: 100%"
          }
        },
        "88d1762760f14bfa8271080da19d8e31": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_c55ebb82486c475295062d7c08ee74e1",
            "max": 3000,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_e55ea6b85dc045a7879fa895de0b280b",
            "value": 3000
          }
        },
        "3d0bd98d7bf84cee8315de1b32f51342": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_0bb5ca3a47b84819901e08288d39aae2",
            "placeholder": "​",
            "style": "IPY_MODEL_227d22b1a636489684891c457f49e56e",
            "value": " 3000/3000 [00:02&lt;00:00, 1485.12 examples/s]"
          }
        },
        "0a036fcd45344c6288d35b8c00a32633": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": "hidden",
            "width": null
          }
        },
        "92a2cc53d1fe4106adcde8e4ce5242e8": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "90360c85b6dd4f098054a89db24c9820": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "c55ebb82486c475295062d7c08ee74e1": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "e55ea6b85dc045a7879fa895de0b280b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "0bb5ca3a47b84819901e08288d39aae2": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "227d22b1a636489684891c457f49e56e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "b33005528ed74071b100705c8905ea0b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_01929af9216742d3a3149efe635f1b23",
              "IPY_MODEL_5b5a0dfc27f944b6af333ac1ae5bbf6a",
              "IPY_MODEL_76e61442aa9f40c8b2e903081c26a4d9"
            ],
            "layout": "IPY_MODEL_9aa0e810949d400c9ab8395afeb1f39a"
          }
        },
        "01929af9216742d3a3149efe635f1b23": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_0e69bd4984d149f8b440bf856377a0ed",
            "placeholder": "​",
            "style": "IPY_MODEL_fbd589917b184c7480bf14bf388db545",
            "value": "Downloading (…)lve/main/config.json: 100%"
          }
        },
        "5b5a0dfc27f944b6af333ac1ae5bbf6a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_9b786d4205e243c380b1808926d23dac",
            "max": 1004,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_f96fbce85e6f485f8b4b0cbdfcac517f",
            "value": 1004
          }
        },
        "76e61442aa9f40c8b2e903081c26a4d9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_d750c4f1dea04c7faf7a1210889a20e4",
            "placeholder": "​",
            "style": "IPY_MODEL_96eb8d18b37c42e2bcfd1f0c212d99a5",
            "value": " 1.00k/1.00k [00:00&lt;00:00, 69.9kB/s]"
          }
        },
        "9aa0e810949d400c9ab8395afeb1f39a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "0e69bd4984d149f8b440bf856377a0ed": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "fbd589917b184c7480bf14bf388db545": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "9b786d4205e243c380b1808926d23dac": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "f96fbce85e6f485f8b4b0cbdfcac517f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "d750c4f1dea04c7faf7a1210889a20e4": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "96eb8d18b37c42e2bcfd1f0c212d99a5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "97439d6a6e8147f7bd1efdfa4b086704": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_1d15e3e6c42140d288006800d6080d03",
              "IPY_MODEL_500be8fe797c43738e856c30bb0cacd6",
              "IPY_MODEL_3daa4d242c4e4fc59ad665c51145a893"
            ],
            "layout": "IPY_MODEL_64afa5dfa0ed4af08b1e0f7e7a05c120"
          }
        },
        "1d15e3e6c42140d288006800d6080d03": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_739ef9b877f949369edf1bf21b397780",
            "placeholder": "​",
            "style": "IPY_MODEL_1b4bf79a639840889cc5c2862575a9f5",
            "value": "Downloading pytorch_model.bin: 100%"
          }
        },
        "500be8fe797c43738e856c30bb0cacd6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_8976d04ad6df4174826ae5f2b1add858",
            "max": 444905837,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_b6e30a8f69d74aeb96c99b14fd497b5f",
            "value": 444905837
          }
        },
        "3daa4d242c4e4fc59ad665c51145a893": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_5851803f8f8048a6857eabe899a611be",
            "placeholder": "​",
            "style": "IPY_MODEL_b2c5b90ecea344019884ba27b8016ded",
            "value": " 445M/445M [00:02&lt;00:00, 162MB/s]"
          }
        },
        "64afa5dfa0ed4af08b1e0f7e7a05c120": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "739ef9b877f949369edf1bf21b397780": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "1b4bf79a639840889cc5c2862575a9f5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "8976d04ad6df4174826ae5f2b1add858": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "b6e30a8f69d74aeb96c99b14fd497b5f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "5851803f8f8048a6857eabe899a611be": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "b2c5b90ecea344019884ba27b8016ded": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "2889e59cfc174efa87dd1f34a8df70ac": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_ba355607e51f4a5bbef960020c6bc407",
              "IPY_MODEL_5e47847e7cdb4039bc09e11c6ade7784",
              "IPY_MODEL_0e94fc424c9a424db66bdc90859bbd17"
            ],
            "layout": "IPY_MODEL_82fb0daf4380427e96010d831a702460"
          }
        },
        "ba355607e51f4a5bbef960020c6bc407": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_bf76f2f518a04e5da9fc9942bb821859",
            "placeholder": "​",
            "style": "IPY_MODEL_701b4f96599e429a982d3064babd4a7f",
            "value": "Running tokenizer on dataset: 100%"
          }
        },
        "5e47847e7cdb4039bc09e11c6ade7784": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_05b5d1168a6540fd951c3fc904025aff",
            "max": 3000,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_20c37db485aa40a4be658581dbc970ff",
            "value": 3000
          }
        },
        "0e94fc424c9a424db66bdc90859bbd17": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_49d24d77165f491b9f74cec7843c2d49",
            "placeholder": "​",
            "style": "IPY_MODEL_d254e8e4d7bb44699e3fdaf47a8eef4c",
            "value": " 3000/3000 [00:01&lt;00:00, 1515.08 examples/s]"
          }
        },
        "82fb0daf4380427e96010d831a702460": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": "hidden",
            "width": null
          }
        },
        "bf76f2f518a04e5da9fc9942bb821859": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "701b4f96599e429a982d3064babd4a7f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "05b5d1168a6540fd951c3fc904025aff": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "20c37db485aa40a4be658581dbc970ff": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "49d24d77165f491b9f74cec7843c2d49": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "d254e8e4d7bb44699e3fdaf47a8eef4c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "source": [
        "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/drive/1NLtpspxcoo467TNaVOHJHC6bQki3ECNK?usp=sharing)"
      ],
      "metadata": {
        "id": "TkhoSWYfbbtV"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "https://www.kaggle.com/code/rhtsingh/utilizing-transformer-representations-efficiently"
      ],
      "metadata": {
        "id": "xuNzJ94yl4IC"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Import library and Setup"
      ],
      "metadata": {
        "id": "63z1zLB5vOsu"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from IPython.display import clear_output\n",
        "!pip install transformers\n",
        "!pip install datasets\n",
        "\n",
        "# jarvisx17/japanese-sentiment-analysis用\n",
        "!pip install fugashi\n",
        "!pip install unidic_lite\n",
        "\n",
        "clear_output()"
      ],
      "metadata": {
        "id": "OXRtmyR-vIHg"
      },
      "execution_count": 1,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import os\n",
        "import sys\n",
        "import pandas as pd\n",
        "import argparse\n",
        "import numpy as np\n",
        "import matplotlib.pyplot as plt\n",
        "import seaborn as sns\n",
        "import random\n",
        "\n",
        "from sklearn.metrics import (accuracy_score, f1_score,\n",
        "                             precision_score, recall_score,)\n",
        "\n",
        "import torch\n",
        "import torch.nn as nn\n",
        "from torch.nn import functional as F\n",
        "from torch.utils.data import DataLoader\n",
        "\n",
        "import time\n",
        "from datetime import datetime\n",
        "import pytz\n",
        "\n",
        "import datasets\n",
        "from datasets import load_dataset\n",
        "from transformers import (AutoModel, AutoTokenizer, AutoConfig,\n",
        "                          AutoModelForSequenceClassification,)\n",
        "from transformers import (Trainer, TrainingArguments,\n",
        "                          DataCollatorWithPadding, EarlyStoppingCallback)\n",
        "from transformers.modeling_outputs import SequenceClassifierOutput\n",
        "\n",
        "from tqdm import tqdm_notebook as tqdm\n",
        "from IPython.display import clear_output\n",
        "import warnings\n",
        "warnings.filterwarnings(\"ignore\")\n",
        "import gc\n",
        "gc.collect();\n",
        "\n",
        "pd.set_option('display.max_rows', 100) \n",
        "pd.set_option('display.max_columns', 100)\n",
        "pd.set_option('display.width', 100)"
      ],
      "metadata": {
        "id": "3CEkmUgRuhDB"
      },
      "execution_count": 2,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# CREATE colaboratory folder\n",
        "PROJECT = 'CDLE_NLP'\n",
        "def SET_Dir(PROJECT):\n",
        "  # CREATE PROJECT FOLD\n",
        "  PROJECTDIR = f'/content/{PROJECT}'\n",
        "  if not os.path.exists(f\"/content/{PROJECT}\"):\n",
        "    !mkdir {PROJECT}\n",
        "  # CREATE INPUT FOLD\n",
        "  INPUTDIR = f'/content/{PROJECT}/input'\n",
        "  if not os.path.exists(f\"/content/{PROJECT}/input\"):\n",
        "    !mkdir {PROJECT}/input\n",
        "  # CREATE OUTPUT FOLD\n",
        "  OUTPUTDIR = f'/content/{PROJECT}/output'\n",
        "  if not os.path.exists(f\"/content/{PROJECT}/output\"):\n",
        "    !mkdir {PROJECT}/output\n",
        "  # CREATE MODEL FOLD\n",
        "  MODELDIR = f'/content/{PROJECT}/model'\n",
        "  if not os.path.exists(f\"/content/{PROJECT}/model\"):\n",
        "    !mkdir {PROJECT}/model\n",
        "  return PROJECTDIR, INPUTDIR, OUTPUTDIR, MODELDIR\n",
        "PROJECTDIR, INPUTDIR, OUTPUTDIR, MODELDIR = SET_Dir(PROJECT)"
      ],
      "metadata": {
        "id": "vSDDOiPJpV5J"
      },
      "execution_count": 3,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# MOUNT GoogleDrive\n",
        "from google.colab import drive \n",
        "drive.mount('/content/drive')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "IpmDM7iDCOHL",
        "outputId": "1a2b4890-d23f-486e-ac0e-18f1229887ed"
      },
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Mounted at /content/drive\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "UyUnzP665zx6"
      },
      "execution_count": 4,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Helper"
      ],
      "metadata": {
        "id": "h8pnv8-m6CaU"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "def seed_everything(seed):\n",
        "    random.seed(seed)\n",
        "    os.environ[\"PYTHONHASHSEED\"] = str(seed)\n",
        "    np.random.seed(seed)\n",
        "    torch.manual_seed(seed)\n",
        "    torch.cuda.manual_seed(seed)\n",
        "    torch.backends.cudnn.deterministic = True\n",
        "\n",
        "def set_output_dir(cfg):\n",
        "  dt_now = datetime.now(pytz.timezone('Asia/Tokyo'))\n",
        "  dttime = str(dt_now.year)+str(dt_now.month)+str(dt_now.day)+\"-\"+str(dt_now.hour)+\"-\"+str(dt_now.minute)+\"-\"+str(dt_now.second)\n",
        "  cfg.output_dir = f\"{dttime}-{cfg.model_path_or_name}-finetuned-emotion\"\n",
        "  print(f'SET output_dir: {cfg.output_dir}')"
      ],
      "metadata": {
        "id": "HkVzeTGR6Fmr"
      },
      "execution_count": 5,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "ebSZtJhs6HI_"
      },
      "execution_count": 5,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Set Config"
      ],
      "metadata": {
        "id": "p-OeCLFS54KF"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "* tip\n",
        "  * casedは大文字と小文字を区別  \n",
        "  * uncasedは大文字と小文字を区別しない"
      ],
      "metadata": {
        "id": "MkP2U2cAYs71"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "class Config:\n",
        "    model_path_or_name = 'jarvisx17/japanese-sentiment-analysis'\n",
        "    output_dir = f\"{model_path_or_name}-finetuned-emotion\"\n",
        "\n",
        "    num_classes = 3\n",
        "    max_seq_length = 512\n",
        "    padding = 'max_length'\n",
        "\n",
        "    num_epochs = 30\n",
        "    early_stopping_patience = 3 # epoc\n",
        "    batch_size = 128\n",
        "    gradient_accumulation_steps = 1 # Memory saving( # 実際のbatch_size = batch_size / gradient_accumulation_steps)\n",
        "    lr = 2e-5\n",
        "    weight_decay = 0.01\n",
        "    warmup_steps = 500\n",
        "    \n",
        "    device = 'cuda' if torch.cuda.is_available() else 'cpu'\n",
        "    num_workers = 1\n",
        "    seed = 72\n",
        "seed_everything(Config.seed)"
      ],
      "metadata": {
        "id": "H4Nqp3ML11OB"
      },
      "execution_count": 6,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "nJVIrOqo5-GK"
      },
      "execution_count": 6,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Load Dataset"
      ],
      "metadata": {
        "id": "XSMM7a6O6PUz"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "dataset = load_dataset(\"tyqiangz/multilingual-sentiments\", \"japanese\")\n",
        "dataset.set_format(type=\"pandas\")\n",
        "\n",
        "train_df = dataset['train'][:].rename(columns={'label': 'labels'}).sample(n=10000) # DEMO\n",
        "valid_df = dataset['validation'][:].rename(columns={'label': 'labels'})\n",
        "test_df = dataset['test'][:].rename(columns={'label': 'labels'})\n",
        "\n",
        "del dataset; gc.collect();\n",
        "clear_output()"
      ],
      "metadata": {
        "id": "c7g0_8vqnC_0"
      },
      "execution_count": 7,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "print(f'train_df len: {len(train_df)}'); display(train_df.head(2));\n",
        "print(f'valid_df len: {len(valid_df)}'); display(valid_df.head(2))\n",
        "print(f'test_df len: {len(test_df)}'); display(test_df.head(2))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 0
        },
        "id": "efSAmmc5nC8_",
        "outputId": "f4e3d356-880a-4640-8fdc-b15f6442cb10"
      },
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "train_df len: 10000\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "                                                    text                source  labels\n",
              "9548   OSをバージョンアップすると、途端に使えなくなる。どんな仕組みで作動するのか知らないが、期待...  amazon_reviews_multi       2\n",
              "28735  設置位置の調整中に気付いたのですが、ダッシュボード上に白い粉が付着していました。息で軽く飛ば...  amazon_reviews_multi       2"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-7835cfd5-2579-4750-831a-ad08c7032297\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>text</th>\n",
              "      <th>source</th>\n",
              "      <th>labels</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>9548</th>\n",
              "      <td>OSをバージョンアップすると、途端に使えなくなる。どんな仕組みで作動するのか知らないが、期待...</td>\n",
              "      <td>amazon_reviews_multi</td>\n",
              "      <td>2</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>28735</th>\n",
              "      <td>設置位置の調整中に気付いたのですが、ダッシュボード上に白い粉が付着していました。息で軽く飛ば...</td>\n",
              "      <td>amazon_reviews_multi</td>\n",
              "      <td>2</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-7835cfd5-2579-4750-831a-ad08c7032297')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-7835cfd5-2579-4750-831a-ad08c7032297 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-7835cfd5-2579-4750-831a-ad08c7032297');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "valid_df len: 3000\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "                                                text                source  labels\n",
              "0  味自体及び吸い心地は良いのだが、不良品が多過ぎる。私の場合５本のうち２本が蒸気も出ず、吸い込...  amazon_reviews_multi       2\n",
              "1             ホームボタン周りの気泡が全く抜けません。 返金をお願いしましたが、断られた。  amazon_reviews_multi       2"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-93b49dea-6d2f-41f9-821e-36b5b7a2985c\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>text</th>\n",
              "      <th>source</th>\n",
              "      <th>labels</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>味自体及び吸い心地は良いのだが、不良品が多過ぎる。私の場合５本のうち２本が蒸気も出ず、吸い込...</td>\n",
              "      <td>amazon_reviews_multi</td>\n",
              "      <td>2</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>ホームボタン周りの気泡が全く抜けません。 返金をお願いしましたが、断られた。</td>\n",
              "      <td>amazon_reviews_multi</td>\n",
              "      <td>2</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-93b49dea-6d2f-41f9-821e-36b5b7a2985c')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-93b49dea-6d2f-41f9-821e-36b5b7a2985c button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-93b49dea-6d2f-41f9-821e-36b5b7a2985c');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "test_df len: 3000\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "                                                text                source  labels\n",
              "0  購入、貼付け後2週間もたたないうちに、上側から剥がれてきて画面から浮いた状態になってしまった...  amazon_reviews_multi       2\n",
              "1  以下の3点の理由により、期待はずれの粗悪品。 ①他の方のレビューにもある通り、天板の外観が掲...  amazon_reviews_multi       2"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-a4eaed1f-a573-4acc-ad60-b02268c01c36\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>text</th>\n",
              "      <th>source</th>\n",
              "      <th>labels</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>購入、貼付け後2週間もたたないうちに、上側から剥がれてきて画面から浮いた状態になってしまった...</td>\n",
              "      <td>amazon_reviews_multi</td>\n",
              "      <td>2</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>以下の3点の理由により、期待はずれの粗悪品。 ①他の方のレビューにもある通り、天板の外観が掲...</td>\n",
              "      <td>amazon_reviews_multi</td>\n",
              "      <td>2</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-a4eaed1f-a573-4acc-ad60-b02268c01c36')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-a4eaed1f-a573-4acc-ad60-b02268c01c36 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-a4eaed1f-a573-4acc-ad60-b02268c01c36');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "wCvviOd-6Sll"
      },
      "execution_count": 8,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Set Tokenize"
      ],
      "metadata": {
        "id": "386HMGIz6SzH"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "def LoadTokenizer(cfg, *, pretrained:bool=True, model_path:str=None):\n",
        "  # https://zenn.dev/hellorusk/articles/7fd588cae5b173\n",
        "  if pretrained:\n",
        "    tokenizer = AutoTokenizer.from_pretrained(cfg.model_path_or_name)\n",
        "  else:\n",
        "    tokenizer = AutoTokenizer.from_pretrained(model_path)\n",
        "\n",
        "  ## infomation\n",
        "  clear_output()\n",
        "  print('*'*20);\n",
        "  if pretrained:\n",
        "    print(f'model name: {cfg.model_path_or_name}')\n",
        "  else:\n",
        "    print(f'model path: {model_path}')\n",
        "  print(f'tokenizer語彙数: {tokenizer.vocab_size}, tokenizer最大長: {tokenizer.model_max_length}');\n",
        "  print(f'tokenizer input_names: {tokenizer.model_input_names}');\n",
        "  print('*'*20);\n",
        "\n",
        "  return tokenizer"
      ],
      "metadata": {
        "id": "Ja-8fg5jqLTD"
      },
      "execution_count": 9,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def create_dataset(df:pd.DataFrame, tokenizer, *, cfg=None, remove_columns:list=None):\n",
        "  def preprocess(examples):\n",
        "    return tokenizer(examples['text'],\n",
        "                     add_special_tokens=True,\n",
        "                     padding=True,\n",
        "                     max_length=cfg.max_seq_length,\n",
        "                     truncation='longest_first',\n",
        "                     return_tensors='pt',\n",
        "                     return_attention_mask=True)\n",
        "    \n",
        "  raw_dataset = datasets.Dataset.from_pandas(df)\n",
        "  return raw_dataset.map(preprocess, batched=True, remove_columns=remove_columns,\n",
        "                         desc=\"Running tokenizer on dataset\",)"
      ],
      "metadata": {
        "id": "9O7yA6D1vuTo"
      },
      "execution_count": 10,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "tokenizer = LoadTokenizer(Config)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 0,
          "referenced_widgets": [
            "0815b713b12f43c783e2901a7738b3a4",
            "1c4bced711c64abd96330f6184673528",
            "965a04241b5d41378b02df443dd528d5",
            "ff4e1c23b9504602968ffb52876cb866",
            "e0fa471119b340a7a8d105ec2b127eaf",
            "07fd2f385e004657ad0504aa694797fa",
            "3147e0cbe84d4c75ab7eb3100b5439d7",
            "5d948a11af534945ade3aa83dcfa978f",
            "52b001d10d424c34a683cb8ffd4df813",
            "505e2c55212f41d29a7450d2707a76c0",
            "71f17654f080472f96bb6d4a708ad6e5",
            "ed8bc585f9f74c45880a8641cfd6cb68",
            "8cfb40b4837740b7b8b31b23ceda6249",
            "2da72d13e7ba43ddae5dd11a6b8c9beb",
            "0e96c411892b42649e40338ffc0f79b8",
            "e9bbd541a00e4cd0b1657dad084f2d9e",
            "0bab86650a9845cbb5ddf9a2c6c52309",
            "48ec0677d3a04a77b3ae1b6e1ef63202",
            "48eb8aebdc1848958e62b7775de17719",
            "282c7333409b4d0997386794b668d4a5",
            "427cd5f36a3d4f31b8133221e0fa39a4",
            "ae98d213cd644b188fb20f290ca175be",
            "9a4e6106b7b34adb8b761adbf7b6d188"
          ]
        },
        "id": "wYRKmnoY2qm1",
        "outputId": "685c7cd4-5bdc-4e54-9823-87a87fc07c46"
      },
      "execution_count": 11,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "********************\n",
            "model name: jarvisx17/japanese-sentiment-analysis\n",
            "tokenizer語彙数: 32768, tokenizer最大長: 1000000000000000019884624838656\n",
            "tokenizer input_names: ['input_ids', 'token_type_ids', 'attention_mask']\n",
            "********************\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "AB_NgvYq6sKd"
      },
      "execution_count": 11,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Create Dataset"
      ],
      "metadata": {
        "id": "JX1BpfxD6sct"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "remove_columns = ['text','source']\n",
        "trn_dataset = create_dataset(train_df, tokenizer, cfg=Config, remove_columns=remove_columns)\n",
        "val_dataset = create_dataset(valid_df, tokenizer, cfg=Config, remove_columns=remove_columns)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 0,
          "referenced_widgets": [
            "fc795c2a39234dc5872a82722596d6df",
            "4bcf544026984b1faf04fc8aa139df5b",
            "db75da41e1d64acb983976356d17b053",
            "5c3f79b9f3a249b4845d6fdc0c6b3074",
            "7e1a29c540c04fadbdd531fd0b3340e9",
            "f8813ff4d2874fdfa2e98b8d02254350",
            "55ba4fa2b283407f9aacdbabeb263dce",
            "aefdb55cc72447968dd259f9e6aebb5a",
            "a67e64e771364e3fa4304a3171deb24b",
            "f20d1409c3ee428c84150f4080ff8e79",
            "a205277548b64e2eaf1540bf535f7b71",
            "37ed2d5f107742768279e25a11af9892",
            "15daf6f5bfbf453987fdf175eb1dd655",
            "88d1762760f14bfa8271080da19d8e31",
            "3d0bd98d7bf84cee8315de1b32f51342",
            "0a036fcd45344c6288d35b8c00a32633",
            "92a2cc53d1fe4106adcde8e4ce5242e8",
            "90360c85b6dd4f098054a89db24c9820",
            "c55ebb82486c475295062d7c08ee74e1",
            "e55ea6b85dc045a7879fa895de0b280b",
            "0bb5ca3a47b84819901e08288d39aae2",
            "227d22b1a636489684891c457f49e56e"
          ]
        },
        "id": "uDBOwduQwedM",
        "outputId": "06cd19d2-bda7-4579-d52b-3f1bbb2b585f"
      },
      "execution_count": 12,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Running tokenizer on dataset:   0%|          | 0/10000 [00:00<?, ? examples/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "fc795c2a39234dc5872a82722596d6df"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Running tokenizer on dataset:   0%|          | 0/3000 [00:00<?, ? examples/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "37ed2d5f107742768279e25a11af9892"
            }
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Model"
      ],
      "metadata": {
        "id": "nqo0iaXS6GpG"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "class CustomModelPooler(nn.Module):\n",
        "\n",
        "  def __init__(self, cfg , *,\n",
        "               dropout:float=0.1, use_multi_layer:int=None,\n",
        "               requires_grad_layer:bool=False, requires_grad_pooler:bool=False,\n",
        "               requires_grad_layer_slit:int=None,\n",
        "               pretrained:bool=True, model_path:str=None,):\n",
        "    super(CustomModelPooler,self).__init__()\n",
        "    self.num_classes = cfg.num_classes\n",
        "    self.dropout = dropout\n",
        "    self.use_multi_layer = use_multi_layer\n",
        "    self.model_path = model_path\n",
        "    self.requires_grad_layer = requires_grad_layer\n",
        "    self.requires_grad_pooler = requires_grad_pooler\n",
        "    self.requires_grad_layer_slit = requires_grad_layer_slit\n",
        "\n",
        "    # model\n",
        "    self.model_path_or_name = cfg.model_path_or_name\n",
        "    self.config = AutoConfig.from_pretrained(cfg.model_path_or_name)\n",
        "    if pretrained:\n",
        "      self.encoder = (AutoModel\n",
        "                      .from_pretrained(cfg.model_path_or_name,\n",
        "                                      config=self.config)\n",
        "                      .to(cfg.device))\n",
        "      # 学習済みモデルの重み凍結\n",
        "      for _, param in self.encoder.named_parameters():\n",
        "        param.requires_grad = False\n",
        "      # 必要に応じて layer層 や pooler層 を学習\n",
        "      if self.requires_grad_layer:\n",
        "        for _, param in self.encoder.encoder.layer[self.requires_grad_layer_slit:].named_parameters():\n",
        "          param.requires_grad = True\n",
        "      if self.requires_grad_pooler:\n",
        "        for _, param in self.encoder.pooler.named_parameters():\n",
        "          param.requires_grad = True\n",
        "    else:\n",
        "      self.encoder = (AutoModel\n",
        "                      .from_pretrained(self.model_path,\n",
        "                                      config=self.config)\n",
        "                      .to(cfg.device))\n",
        "    # head\n",
        "    self.classifier = nn.Sequential(\n",
        "        nn.Dropout(dropout),\n",
        "        nn.Linear(self.encoder.config.hidden_size, self.num_classes),\n",
        "    )\n",
        "    ## infomation\n",
        "    clear_output()\n",
        "    print('*'*20);\n",
        "    if pretrained:\n",
        "      print(f'model name: {cfg.model_path_or_name}, device: {cfg.device}');\n",
        "    else:\n",
        "      print(f'model path: {self.model_path}, device: {cfg.device}');\n",
        "    print(f'model hidden_size: {self.config.hidden_size}, num_hidden_layers: {self.config.num_hidden_layers}');\n",
        "    print(f'=> num_classes: {self.num_classes}');\n",
        "    if self.requires_grad_layer or self.requires_grad_pooler:\n",
        "      print('=> requires_grad is True list')\n",
        "      for name, param in self.encoder.named_parameters():\n",
        "          if param.requires_grad : print(f'   {name}');\n",
        "    else:\n",
        "      print('=> requires_grad is ALL False')\n",
        "    print('*'*20);\n",
        "\n",
        "  def forward(self, input_ids=None, attention_mask=None, token_type_ids=None, labels=None, **kwargs):\n",
        "    # body\n",
        "    outputs = self.encoder(input_ids=input_ids,\n",
        "                           attention_mask=attention_mask,\n",
        "                           token_type_ids=token_type_ids,\n",
        "                           output_hidden_states=False)\n",
        "    # head(input: pooler_output)\n",
        "    head_input = outputs.pooler_output\n",
        "    logits = self.classifier(head_input)\n",
        "\n",
        "    loss = None\n",
        "    if labels is not None:\n",
        "      loss_fct = nn.CrossEntropyLoss()\n",
        "      loss = loss_fct(logits.view(-1, self.num_classes), labels.view(-1))\n",
        "\n",
        "    return SequenceClassifierOutput(loss=loss, logits=logits)\n",
        "\n",
        "class CustomModelLastHiddenLayer(nn.Module):\n",
        "\n",
        "  def __init__(self, cfg , *,\n",
        "               dropout:float=0.1, use_multi_layer:int=None,\n",
        "               requires_grad_layer:bool=False, requires_grad_pooler:bool=False,\n",
        "               requires_grad_layer_slit:int=None,\n",
        "               pretrained:bool=True, model_path:str=None,):\n",
        "    super(CustomModelLastHiddenLayer,self).__init__()\n",
        "    self.num_classes = cfg.num_classes\n",
        "    self.dropout = dropout\n",
        "    self.use_multi_layer = use_multi_layer\n",
        "    self.model_path = model_path\n",
        "    self.requires_grad_layer = requires_grad_layer\n",
        "    self.requires_grad_pooler = requires_grad_pooler\n",
        "    self.requires_grad_layer_slit = requires_grad_layer_slit\n",
        "\n",
        "    # model\n",
        "    self.model_path_or_name = cfg.model_path_or_name\n",
        "    self.config = AutoConfig.from_pretrained(cfg.model_path_or_name)\n",
        "    if pretrained:\n",
        "      self.encoder = (AutoModel\n",
        "                      .from_pretrained(cfg.model_path_or_name,\n",
        "                                      config=self.config)\n",
        "                      .to(cfg.device))\n",
        "      # 学習済みモデルの重み凍結\n",
        "      for _, param in self.encoder.named_parameters():\n",
        "        param.requires_grad = False\n",
        "      # 必要に応じて layer層 や pooler層 を学習\n",
        "      if self.requires_grad_layer:\n",
        "        for _, param in self.encoder.encoder.layer[self.requires_grad_layer_slit:].named_parameters():\n",
        "          param.requires_grad = True\n",
        "      if self.requires_grad_pooler:\n",
        "        for _, param in self.encoder.pooler.named_parameters():\n",
        "          param.requires_grad = True\n",
        "    else:\n",
        "      self.encoder = (AutoModel\n",
        "                      .from_pretrained(self.model_path,\n",
        "                                      config=self.config)\n",
        "                      .to(cfg.device))\n",
        "    # head\n",
        "    self.classifier = nn.Sequential(\n",
        "        nn.Dropout(dropout),\n",
        "        nn.Linear(self.encoder.config.hidden_size, self.num_classes),\n",
        "    )\n",
        "    ## infomation\n",
        "    clear_output()\n",
        "    print('*'*20);\n",
        "    if pretrained:\n",
        "      print(f'model name: {cfg.model_path_or_name}, device: {cfg.device}');\n",
        "    else:\n",
        "      print(f'model path: {self.model_path}, device: {cfg.device}');\n",
        "    print(f'model hidden_size: {self.config.hidden_size}, num_hidden_layers: {self.config.num_hidden_layers}');\n",
        "    print(f'=> num_classes: {self.num_classes}');\n",
        "    if self.requires_grad_layer or self.requires_grad_pooler:\n",
        "      print('=> requires_grad is True list')\n",
        "      for name, param in self.encoder.named_parameters():\n",
        "          if param.requires_grad : print(f'   {name}');\n",
        "    else:\n",
        "      print('=> requires_grad is ALL False')\n",
        "    print('*'*20);\n",
        "\n",
        "  def forward(self, input_ids=None, attention_mask=None, token_type_ids=None, labels=None, **kwargs):\n",
        "    # body\n",
        "    outputs = self.encoder(input_ids=input_ids,\n",
        "                           attention_mask=attention_mask,\n",
        "                           token_type_ids=token_type_ids,\n",
        "                           output_hidden_states=False)\n",
        "    # head(input: [CLS] feature of last_hidden_state)\n",
        "    head_input = outputs.last_hidden_state[:, 0]\n",
        "    logits = self.classifier(head_input)\n",
        "\n",
        "    loss = None\n",
        "    if labels is not None:\n",
        "      loss_fct = nn.CrossEntropyLoss()\n",
        "      loss = loss_fct(logits.view(-1, self.num_classes), labels.view(-1))\n",
        "\n",
        "    return SequenceClassifierOutput(loss=loss, logits=logits)\n",
        "\n",
        "class CustomModelMultiHiddenLayer(nn.Module):\n",
        "\n",
        "  def __init__(self, cfg , *,\n",
        "               dropout:float=0.1, use_multi_layer:int=None,\n",
        "               requires_grad_layer:bool=False, requires_grad_pooler:bool=False,\n",
        "               requires_grad_layer_slit:int=None,\n",
        "               pretrained:bool=True, model_path:str=None,):\n",
        "    super(CustomModelMultiHiddenLayer,self).__init__()\n",
        "    self.num_classes = cfg.num_classes\n",
        "    self.dropout = dropout\n",
        "    self.use_multi_layer = use_multi_layer\n",
        "    self.model_path = model_path\n",
        "    self.requires_grad_layer = requires_grad_layer\n",
        "    self.requires_grad_pooler = requires_grad_pooler\n",
        "    self.requires_grad_layer_slit = requires_grad_layer_slit\n",
        "\n",
        "    # model\n",
        "    self.model_path_or_name = cfg.model_path_or_name\n",
        "    self.config = AutoConfig.from_pretrained(cfg.model_path_or_name)\n",
        "    if pretrained:\n",
        "      self.encoder = (AutoModel\n",
        "                      .from_pretrained(cfg.model_path_or_name,\n",
        "                                      config=self.config)\n",
        "                      .to(cfg.device))\n",
        "      # 学習済みモデルの重み凍結\n",
        "      for _, param in self.encoder.named_parameters():\n",
        "        param.requires_grad = False\n",
        "      # 必要に応じて layer層 や pooler層 を学習\n",
        "      if self.requires_grad_layer:\n",
        "        for _, param in self.encoder.encoder.layer[self.requires_grad_layer_slit:].named_parameters():\n",
        "          param.requires_grad = True\n",
        "      if self.requires_grad_pooler:\n",
        "        for _, param in self.encoder.pooler.named_parameters():\n",
        "          param.requires_grad = True\n",
        "    else:\n",
        "      self.encoder = (AutoModel\n",
        "                      .from_pretrained(self.model_path,\n",
        "                                      config=self.config)\n",
        "                      .to(cfg.device))\n",
        "    # head\n",
        "    self.classifier = nn.Sequential(\n",
        "        nn.Dropout(self.dropout),\n",
        "        nn.Linear(self.encoder.config.hidden_size*(self.use_multi_layer*-1), self.num_classes),\n",
        "    )\n",
        "    ## infomation\n",
        "    clear_output()\n",
        "    print('*'*20);\n",
        "    if pretrained:\n",
        "      print(f'model name: {cfg.model_path_or_name}, device: {cfg.device}');\n",
        "    else:\n",
        "      print(f'model path: {self.model_path}, device: {cfg.device}');\n",
        "    print(f'model hidden_size: {self.config.hidden_size}, num_hidden_layers: {self.config.num_hidden_layers}');\n",
        "    print(f'=> num_classes: {self.num_classes}');\n",
        "    if self.requires_grad_layer or self.requires_grad_pooler:\n",
        "      print('=> requires_grad is True list')\n",
        "      for name, param in self.encoder.named_parameters():\n",
        "          if param.requires_grad : print(f'   {name}');\n",
        "    else:\n",
        "      print('=> requires_grad is ALL False')\n",
        "    print('*'*20);\n",
        "\n",
        "  def forward(self, input_ids=None, attention_mask=None, token_type_ids=None, labels=None, **kwargs):\n",
        "    # body\n",
        "    outputs = self.encoder(input_ids=input_ids,\n",
        "                           attention_mask=attention_mask,\n",
        "                           token_type_ids=token_type_ids,\n",
        "                           output_hidden_states=True)\n",
        "    # head(input: [CLS] feature of multiple hidden_states)\n",
        "    head_input = torch.cat(outputs.hidden_states[self.use_multi_layer:], 2)[:,0,:]\n",
        "    logits = self.classifier(head_input)\n",
        "\n",
        "    loss = None\n",
        "    if labels is not None:\n",
        "      loss_fct = nn.CrossEntropyLoss()\n",
        "      loss = loss_fct(logits.view(-1, self.num_classes), labels.view(-1))\n",
        "\n",
        "    return SequenceClassifierOutput(loss=loss, logits=logits)\n",
        "\n",
        "\n",
        "def masked_mean(hidden_state, mask):\n",
        "    mask = mask[:, :, np.newaxis].float()\n",
        "    hidden_state_sum = torch.sum(hidden_state * mask, 1)\n",
        "    mask_sum = torch.clamp(torch.sum(mask, 1), min=1e-5)\n",
        "    return hidden_state_sum / mask_sum\n",
        "def masked_max(hidden_state, mask):\n",
        "    mask = (1 - mask[:, :, np.newaxis]).float()\n",
        "    hidden_state = hidden_state - mask * 1e3\n",
        "    return torch.max(hidden_state, 1)[0]\n",
        "\n",
        "class CustomModelLastHiddenLayerMeanPooling(nn.Module):\n",
        "\n",
        "  def __init__(self, cfg , *,\n",
        "               dropout:float=0.1, use_multi_layer:int=None,\n",
        "               requires_grad_layer:bool=False, requires_grad_pooler:bool=False,\n",
        "               requires_grad_layer_slit:int=None,\n",
        "               pretrained:bool=True, model_path:str=None,):\n",
        "    super(CustomModelLastHiddenLayerMeanPooling,self).__init__()\n",
        "    self.num_classes = cfg.num_classes\n",
        "    self.dropout = dropout\n",
        "    self.use_multi_layer = use_multi_layer\n",
        "    self.model_path = model_path\n",
        "    self.requires_grad_layer = requires_grad_layer\n",
        "    self.requires_grad_pooler = requires_grad_pooler\n",
        "    self.requires_grad_layer_slit = requires_grad_layer_slit\n",
        "\n",
        "    # model\n",
        "    self.model_path_or_name = cfg.model_path_or_name\n",
        "    self.config = AutoConfig.from_pretrained(cfg.model_path_or_name)\n",
        "    if pretrained:\n",
        "      self.encoder = (AutoModel\n",
        "                      .from_pretrained(cfg.model_path_or_name,\n",
        "                                      config=self.config)\n",
        "                      .to(cfg.device))\n",
        "      # 学習済みモデルの重み凍結\n",
        "      for _, param in self.encoder.named_parameters():\n",
        "        param.requires_grad = False\n",
        "      # 必要に応じて layer層 や pooler層 を学習\n",
        "      if self.requires_grad_layer:\n",
        "        for _, param in self.encoder.encoder.layer[self.requires_grad_layer_slit:].named_parameters():\n",
        "          param.requires_grad = True\n",
        "      if self.requires_grad_pooler:\n",
        "        for _, param in self.encoder.pooler.named_parameters():\n",
        "          param.requires_grad = True\n",
        "    else:\n",
        "      self.encoder = (AutoModel\n",
        "                      .from_pretrained(self.model_path,\n",
        "                                      config=self.config)\n",
        "                      .to(cfg.device))\n",
        "    # head\n",
        "    self.classifier = nn.Sequential(\n",
        "        nn.Dropout(self.dropout),\n",
        "        nn.Linear(self.encoder.config.hidden_size, self.num_classes),\n",
        "    )\n",
        "    ## infomation\n",
        "    clear_output()\n",
        "    print('*'*20);\n",
        "    if pretrained:\n",
        "      print(f'model name: {cfg.model_path_or_name}, device: {cfg.device}');\n",
        "    else:\n",
        "      print(f'model path: {self.model_path}, device: {cfg.device}');\n",
        "    print(f'model hidden_size: {self.config.hidden_size}, num_hidden_layers: {self.config.num_hidden_layers}');\n",
        "    print(f'=> num_classes: {self.num_classes}');\n",
        "    if self.requires_grad_layer or self.requires_grad_pooler:\n",
        "      print('=> requires_grad is True list')\n",
        "      for name, param in self.encoder.named_parameters():\n",
        "          if param.requires_grad : print(f'   {name}');\n",
        "    else:\n",
        "      print('=> requires_grad is ALL False')\n",
        "    print('*'*20);\n",
        "\n",
        "  def forward(self, input_ids=None, attention_mask=None, token_type_ids=None, labels=None, **kwargs):\n",
        "    # body\n",
        "    outputs = self.encoder(input_ids=input_ids,\n",
        "                           attention_mask=attention_mask,\n",
        "                           token_type_ids=token_type_ids,\n",
        "                           output_hidden_states=False)\n",
        "    # head(input: mean [CLS] feature of last_hidden_state )\n",
        "    head_input = masked_mean(outputs.last_hidden_state, attention_mask)\n",
        "    logits = self.classifier(head_input)\n",
        "\n",
        "    loss = None\n",
        "    if labels is not None:\n",
        "      loss_fct = nn.CrossEntropyLoss()\n",
        "      loss = loss_fct(logits.view(-1, self.num_classes), labels.view(-1))\n",
        "\n",
        "    return SequenceClassifierOutput(loss=loss, logits=logits)\n",
        "\n",
        "class CustomModelLastHiddenLayerMaxPooling(nn.Module):\n",
        "\n",
        "  def __init__(self, cfg , *,\n",
        "               dropout:float=0.1, use_multi_layer:int=None,\n",
        "               requires_grad_layer:bool=False, requires_grad_pooler:bool=False,\n",
        "               requires_grad_layer_slit:int=None,\n",
        "               pretrained:bool=True, model_path:str=None,):\n",
        "    super(CustomModelLastHiddenLayerMaxPooling,self).__init__()\n",
        "    self.num_classes = cfg.num_classes\n",
        "    self.dropout = dropout\n",
        "    self.use_multi_layer = use_multi_layer\n",
        "    self.model_path = model_path\n",
        "    self.requires_grad_layer = requires_grad_layer\n",
        "    self.requires_grad_pooler = requires_grad_pooler\n",
        "    self.requires_grad_layer_slit = requires_grad_layer_slit\n",
        "\n",
        "    # model\n",
        "    self.model_path_or_name = cfg.model_path_or_name\n",
        "    self.config = AutoConfig.from_pretrained(cfg.model_path_or_name)\n",
        "    if pretrained:\n",
        "      self.encoder = (AutoModel\n",
        "                      .from_pretrained(cfg.model_path_or_name,\n",
        "                                      config=self.config)\n",
        "                      .to(cfg.device))\n",
        "      # 学習済みモデルの重み凍結\n",
        "      for _, param in self.encoder.named_parameters():\n",
        "        param.requires_grad = False\n",
        "      # 必要に応じて layer層 や pooler層 を学習\n",
        "      if self.requires_grad_layer:\n",
        "        for _, param in self.encoder.encoder.layer[self.requires_grad_layer_slit:].named_parameters():\n",
        "          param.requires_grad = True\n",
        "      if self.requires_grad_pooler:\n",
        "        for _, param in self.encoder.pooler.named_parameters():\n",
        "          param.requires_grad = True\n",
        "    else:\n",
        "      self.encoder = (AutoModel\n",
        "                      .from_pretrained(self.model_path,\n",
        "                                      config=self.config)\n",
        "                      .to(cfg.device))\n",
        "    # head\n",
        "    self.classifier = nn.Sequential(\n",
        "        nn.Dropout(self.dropout),\n",
        "        nn.Linear(self.encoder.config.hidden_size, self.num_classes),\n",
        "    )\n",
        "    ## infomation\n",
        "    clear_output()\n",
        "    print('*'*20);\n",
        "    if pretrained:\n",
        "      print(f'model name: {cfg.model_path_or_name}, device: {cfg.device}');\n",
        "    else:\n",
        "      print(f'model path: {self.model_path}, device: {cfg.device}');\n",
        "    print(f'model hidden_size: {self.config.hidden_size}, num_hidden_layers: {self.config.num_hidden_layers}');\n",
        "    print(f'=> num_classes: {self.num_classes}');\n",
        "    if self.requires_grad_layer or self.requires_grad_pooler:\n",
        "      print('=> requires_grad is True list')\n",
        "      for name, param in self.encoder.named_parameters():\n",
        "          if param.requires_grad : print(f'   {name}');\n",
        "    else:\n",
        "      print('=> requires_grad is ALL False')\n",
        "    print('*'*20);\n",
        "\n",
        "  def forward(self, input_ids=None, attention_mask=None, token_type_ids=None, labels=None, **kwargs):\n",
        "    # body\n",
        "    outputs = self.encoder(input_ids=input_ids,\n",
        "                           attention_mask=attention_mask,\n",
        "                           token_type_ids=token_type_ids,\n",
        "                           output_hidden_states=False)\n",
        "    # head(input: maximum [CLS] feature of last_hidden_state )\n",
        "    head_input = masked_max(outputs.last_hidden_state, attention_mask)\n",
        "    logits = self.classifier(head_input)\n",
        "\n",
        "    loss = None\n",
        "    if labels is not None:\n",
        "      loss_fct = nn.CrossEntropyLoss()\n",
        "      loss = loss_fct(logits.view(-1, self.num_classes), labels.view(-1))\n",
        "\n",
        "    return SequenceClassifierOutput(loss=loss, logits=logits)\n",
        "\n",
        "class CustomModelLastHiddenLayerMeanMaxPooling(nn.Module):\n",
        "\n",
        "  def __init__(self, cfg , *,\n",
        "               dropout:float=0.1, use_multi_layer:int=None,\n",
        "               requires_grad_layer:bool=False, requires_grad_pooler:bool=False,\n",
        "               requires_grad_layer_slit:int=None,\n",
        "               pretrained:bool=True, model_path:str=None,):\n",
        "    super(CustomModelLastHiddenLayerMeanMaxPooling,self).__init__()\n",
        "    self.num_classes = cfg.num_classes\n",
        "    self.dropout = dropout\n",
        "    self.use_multi_layer = use_multi_layer\n",
        "    self.model_path = model_path\n",
        "    self.requires_grad_layer = requires_grad_layer\n",
        "    self.requires_grad_pooler = requires_grad_pooler\n",
        "    self.requires_grad_layer_slit = requires_grad_layer_slit\n",
        "\n",
        "    # model\n",
        "    self.model_path_or_name = cfg.model_path_or_name\n",
        "    self.config = AutoConfig.from_pretrained(cfg.model_path_or_name)\n",
        "    if pretrained:\n",
        "      self.encoder = (AutoModel\n",
        "                      .from_pretrained(cfg.model_path_or_name,\n",
        "                                      config=self.config)\n",
        "                      .to(cfg.device))\n",
        "      # 学習済みモデルの重み凍結\n",
        "      for _, param in self.encoder.named_parameters():\n",
        "        param.requires_grad = False\n",
        "      # 必要に応じて layer層 や pooler層 を学習\n",
        "      if self.requires_grad_layer:\n",
        "        for _, param in self.encoder.encoder.layer[self.requires_grad_layer_slit:].named_parameters():\n",
        "          param.requires_grad = True\n",
        "      if self.requires_grad_pooler:\n",
        "        for _, param in self.encoder.pooler.named_parameters():\n",
        "          param.requires_grad = True\n",
        "    else:\n",
        "      self.encoder = (AutoModel\n",
        "                      .from_pretrained(self.model_path,\n",
        "                                      config=self.config)\n",
        "                      .to(cfg.device))\n",
        "    # head\n",
        "    self.classifier = nn.Sequential(\n",
        "        nn.Dropout(self.dropout),\n",
        "        nn.Linear(self.encoder.config.hidden_size*2, self.num_classes),\n",
        "    )\n",
        "    ## infomation\n",
        "    clear_output()\n",
        "    print('*'*20);\n",
        "    if pretrained:\n",
        "      print(f'model name: {cfg.model_path_or_name}, device: {cfg.device}');\n",
        "    else:\n",
        "      print(f'model path: {self.model_path}, device: {cfg.device}');\n",
        "    print(f'model hidden_size: {self.config.hidden_size}, num_hidden_layers: {self.config.num_hidden_layers}');\n",
        "    print(f'=> num_classes: {self.num_classes}');\n",
        "    if self.requires_grad_layer or self.requires_grad_pooler:\n",
        "      print('=> requires_grad is True list')\n",
        "      for name, param in self.encoder.named_parameters():\n",
        "          if param.requires_grad : print(f'   {name}');\n",
        "    else:\n",
        "      print('=> requires_grad is ALL False')\n",
        "    print('*'*20);\n",
        "\n",
        "  def forward(self, input_ids=None, attention_mask=None, token_type_ids=None, labels=None, **kwargs):\n",
        "    # body\n",
        "    outputs = self.encoder(input_ids=input_ids,\n",
        "                           attention_mask=attention_mask,\n",
        "                           token_type_ids=token_type_ids,\n",
        "                           output_hidden_states=False)\n",
        "    # head(input: mean and maximum [CLS] feature of last_hidden_state )\n",
        "    head_input = torch.cat(\n",
        "                        (\n",
        "                            masked_mean(outputs.last_hidden_state,attention_mask),\n",
        "                            masked_max(outputs.last_hidden_state, attention_mask)\n",
        "                        ), 1)\n",
        "    logits = self.classifier(head_input)\n",
        "\n",
        "    loss = None\n",
        "    if labels is not None:\n",
        "      loss_fct = nn.CrossEntropyLoss()\n",
        "      loss = loss_fct(logits.view(-1, self.num_classes), labels.view(-1))\n",
        "\n",
        "    return SequenceClassifierOutput(loss=loss, logits=logits)"
      ],
      "metadata": {
        "id": "vhfVxpSGug7k"
      },
      "execution_count": 13,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "uL4lgCi5nDBu"
      },
      "execution_count": 13,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Set Train Args"
      ],
      "metadata": {
        "id": "zA5c3NYM6Xcz"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "def compute_metrics(pred):\n",
        "  labels = pred.label_ids\n",
        "  preds = pred.predictions.argmax(-1)\n",
        "  acc = accuracy_score(labels, preds)\n",
        "  f1 = f1_score(labels, preds, average='weighted')\n",
        "  precision = precision_score(labels, preds, average='weighted')\n",
        "  recall = recall_score(labels, preds, average='weighted')\n",
        "  return {'accuracy': acc, 'f1': f1, 'precision': precision, 'recall': recall,}"
      ],
      "metadata": {
        "id": "C9KEed3o2t77"
      },
      "execution_count": 14,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def set_args(cfg):\n",
        "  batch_size_ = cfg.batch_size // cfg.gradient_accumulation_steps # 実際のbatch_size\n",
        "  logging_steps = len(trn_dataset) // batch_size_\n",
        "\n",
        "  set_output_dir(cfg)\n",
        "  training_args = TrainingArguments(output_dir=MODELDIR+'/'+cfg.output_dir,# 出力場所\n",
        "                                    label_names=['labels'],        # targetLabel( Default: labels )\n",
        "                                    num_train_epochs=cfg.num_epochs,\n",
        "                                    gradient_accumulation_steps=cfg.gradient_accumulation_steps,\n",
        "                                    per_device_train_batch_size=batch_size_,\n",
        "                                    per_device_eval_batch_size=batch_size_,\n",
        "                                    learning_rate=cfg.lr,          # Default: 5e-5\n",
        "                                    lr_scheduler_type='linear',    # 減衰設定( Default: linear )\n",
        "                                    weight_decay=cfg.weight_decay, # 重み減衰の強さ\n",
        "                                    warmup_steps=cfg.warmup_steps, # 学習率スケジューラのウォームアップステップ数\n",
        "                                    seed=cfg.seed,\n",
        "                                    fp16=True, # Automatic Mixed Precision (AMP)\n",
        "                                    evaluation_strategy='epoch', # 評価タイミング\n",
        "                                    save_strategy='epoch',       # 保存タイミング\n",
        "                                    logging_strategy='epoch',    # loggingタイミング\n",
        "                                    # eval_steps=logging_steps,\n",
        "                                    # save_steps=logging_steps,\n",
        "                                    # logging_steps=logging_steps, \n",
        "                                    save_total_limit = 1,        # output_dirに残すチェックポイントの数\n",
        "                                    metric_for_best_model='f1',  # EarlyStoppingの判断基準(compute_metrics)\n",
        "                                    load_best_model_at_end=True, # EarlyStoppingを使用するならTrue\n",
        "                                    disable_tqdm=False,\n",
        "                                    push_to_hub=False, \n",
        "                                    log_level='error',\n",
        "                                    report_to=['none'],\n",
        "                                    )\n",
        "  return training_args, training_args.output_dir"
      ],
      "metadata": {
        "id": "s_fILAcinlFo"
      },
      "execution_count": 15,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Training"
      ],
      "metadata": {
        "id": "GN_xtOpN6ir2"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "model = CustomModelPooler(Config,\n",
        "                          requires_grad_layer=True, requires_grad_pooler=True,\n",
        "                          requires_grad_layer_slit=-2)\n",
        "training_args, save_path = set_args(Config)\n",
        "\n",
        "trainer = Trainer(model=model, args=training_args,\n",
        "                  compute_metrics=compute_metrics,\n",
        "                  train_dataset=trn_dataset,\n",
        "                  eval_dataset=val_dataset,\n",
        "                  tokenizer=tokenizer,\n",
        "                  callbacks=[EarlyStoppingCallback(\n",
        "                      early_stopping_patience=Config.early_stopping_patience\n",
        "                      )],\n",
        "                  data_collator=DataCollatorWithPadding(\n",
        "                      tokenizer=tokenizer, padding='longest',\n",
        "                      ),\n",
        "                  )\n",
        "\n",
        "trn_result = trainer.train()\n",
        "val_result = trainer.evaluate(eval_dataset=val_dataset)\n",
        "display(pd.DataFrame(val_result, index=[0]))\n",
        "\n",
        "trainer.save_model(save_path)\n",
        "save_path_1 = save_path\n",
        "\n",
        "torch.cuda.empty_cache(); gc.collect();"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000,
          "referenced_widgets": [
            "b33005528ed74071b100705c8905ea0b",
            "01929af9216742d3a3149efe635f1b23",
            "5b5a0dfc27f944b6af333ac1ae5bbf6a",
            "76e61442aa9f40c8b2e903081c26a4d9",
            "9aa0e810949d400c9ab8395afeb1f39a",
            "0e69bd4984d149f8b440bf856377a0ed",
            "fbd589917b184c7480bf14bf388db545",
            "9b786d4205e243c380b1808926d23dac",
            "f96fbce85e6f485f8b4b0cbdfcac517f",
            "d750c4f1dea04c7faf7a1210889a20e4",
            "96eb8d18b37c42e2bcfd1f0c212d99a5",
            "97439d6a6e8147f7bd1efdfa4b086704",
            "1d15e3e6c42140d288006800d6080d03",
            "500be8fe797c43738e856c30bb0cacd6",
            "3daa4d242c4e4fc59ad665c51145a893",
            "64afa5dfa0ed4af08b1e0f7e7a05c120",
            "739ef9b877f949369edf1bf21b397780",
            "1b4bf79a639840889cc5c2862575a9f5",
            "8976d04ad6df4174826ae5f2b1add858",
            "b6e30a8f69d74aeb96c99b14fd497b5f",
            "5851803f8f8048a6857eabe899a611be",
            "b2c5b90ecea344019884ba27b8016ded"
          ]
        },
        "id": "NP3F9DuEnLqE",
        "outputId": "e4b9b867-3bae-4260-cccf-4280b2014b1a"
      },
      "execution_count": 16,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "********************\n",
            "model name: jarvisx17/japanese-sentiment-analysis, device: cuda\n",
            "model hidden_size: 768, num_hidden_layers: 12\n",
            "=> num_classes: 3\n",
            "=> requires_grad is True list\n",
            "   encoder.layer.10.attention.self.query.weight\n",
            "   encoder.layer.10.attention.self.query.bias\n",
            "   encoder.layer.10.attention.self.key.weight\n",
            "   encoder.layer.10.attention.self.key.bias\n",
            "   encoder.layer.10.attention.self.value.weight\n",
            "   encoder.layer.10.attention.self.value.bias\n",
            "   encoder.layer.10.attention.output.dense.weight\n",
            "   encoder.layer.10.attention.output.dense.bias\n",
            "   encoder.layer.10.attention.output.LayerNorm.weight\n",
            "   encoder.layer.10.attention.output.LayerNorm.bias\n",
            "   encoder.layer.10.intermediate.dense.weight\n",
            "   encoder.layer.10.intermediate.dense.bias\n",
            "   encoder.layer.10.output.dense.weight\n",
            "   encoder.layer.10.output.dense.bias\n",
            "   encoder.layer.10.output.LayerNorm.weight\n",
            "   encoder.layer.10.output.LayerNorm.bias\n",
            "   encoder.layer.11.attention.self.query.weight\n",
            "   encoder.layer.11.attention.self.query.bias\n",
            "   encoder.layer.11.attention.self.key.weight\n",
            "   encoder.layer.11.attention.self.key.bias\n",
            "   encoder.layer.11.attention.self.value.weight\n",
            "   encoder.layer.11.attention.self.value.bias\n",
            "   encoder.layer.11.attention.output.dense.weight\n",
            "   encoder.layer.11.attention.output.dense.bias\n",
            "   encoder.layer.11.attention.output.LayerNorm.weight\n",
            "   encoder.layer.11.attention.output.LayerNorm.bias\n",
            "   encoder.layer.11.intermediate.dense.weight\n",
            "   encoder.layer.11.intermediate.dense.bias\n",
            "   encoder.layer.11.output.dense.weight\n",
            "   encoder.layer.11.output.dense.bias\n",
            "   encoder.layer.11.output.LayerNorm.weight\n",
            "   encoder.layer.11.output.LayerNorm.bias\n",
            "   pooler.dense.weight\n",
            "   pooler.dense.bias\n",
            "********************\n",
            "SET output_dir: 202349-22-14-44-jarvisx17/japanese-sentiment-analysis-finetuned-emotion\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              "\n",
              "    <div>\n",
              "      \n",
              "      <progress value='1106' max='2370' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
              "      [1106/2370 09:47 < 11:12, 1.88 it/s, Epoch 14/30]\n",
              "    </div>\n",
              "    <table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              " <tr style=\"text-align: left;\">\n",
              "      <th>Epoch</th>\n",
              "      <th>Training Loss</th>\n",
              "      <th>Validation Loss</th>\n",
              "      <th>Accuracy</th>\n",
              "      <th>F1</th>\n",
              "      <th>Precision</th>\n",
              "      <th>Recall</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <td>1</td>\n",
              "      <td>0.968900</td>\n",
              "      <td>0.796381</td>\n",
              "      <td>0.666000</td>\n",
              "      <td>0.574176</td>\n",
              "      <td>0.678792</td>\n",
              "      <td>0.666000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2</td>\n",
              "      <td>0.684400</td>\n",
              "      <td>0.588844</td>\n",
              "      <td>0.746333</td>\n",
              "      <td>0.729831</td>\n",
              "      <td>0.738964</td>\n",
              "      <td>0.746333</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3</td>\n",
              "      <td>0.596600</td>\n",
              "      <td>0.553072</td>\n",
              "      <td>0.763333</td>\n",
              "      <td>0.756831</td>\n",
              "      <td>0.757134</td>\n",
              "      <td>0.763333</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>4</td>\n",
              "      <td>0.565700</td>\n",
              "      <td>0.542115</td>\n",
              "      <td>0.774333</td>\n",
              "      <td>0.765981</td>\n",
              "      <td>0.768606</td>\n",
              "      <td>0.774333</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>5</td>\n",
              "      <td>0.549200</td>\n",
              "      <td>0.510240</td>\n",
              "      <td>0.784667</td>\n",
              "      <td>0.776557</td>\n",
              "      <td>0.780313</td>\n",
              "      <td>0.784667</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>6</td>\n",
              "      <td>0.531500</td>\n",
              "      <td>0.495036</td>\n",
              "      <td>0.788667</td>\n",
              "      <td>0.783990</td>\n",
              "      <td>0.784228</td>\n",
              "      <td>0.788667</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>7</td>\n",
              "      <td>0.525700</td>\n",
              "      <td>0.495073</td>\n",
              "      <td>0.794333</td>\n",
              "      <td>0.789749</td>\n",
              "      <td>0.790626</td>\n",
              "      <td>0.794333</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>8</td>\n",
              "      <td>0.506600</td>\n",
              "      <td>0.501105</td>\n",
              "      <td>0.793667</td>\n",
              "      <td>0.787726</td>\n",
              "      <td>0.789657</td>\n",
              "      <td>0.793667</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>9</td>\n",
              "      <td>0.500200</td>\n",
              "      <td>0.483902</td>\n",
              "      <td>0.800000</td>\n",
              "      <td>0.796360</td>\n",
              "      <td>0.796196</td>\n",
              "      <td>0.800000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>10</td>\n",
              "      <td>0.495200</td>\n",
              "      <td>0.495991</td>\n",
              "      <td>0.800333</td>\n",
              "      <td>0.797570</td>\n",
              "      <td>0.797628</td>\n",
              "      <td>0.800333</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>11</td>\n",
              "      <td>0.477000</td>\n",
              "      <td>0.479190</td>\n",
              "      <td>0.806333</td>\n",
              "      <td>0.805723</td>\n",
              "      <td>0.805410</td>\n",
              "      <td>0.806333</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>12</td>\n",
              "      <td>0.470300</td>\n",
              "      <td>0.503442</td>\n",
              "      <td>0.797333</td>\n",
              "      <td>0.790656</td>\n",
              "      <td>0.794056</td>\n",
              "      <td>0.797333</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>13</td>\n",
              "      <td>0.462400</td>\n",
              "      <td>0.482490</td>\n",
              "      <td>0.805333</td>\n",
              "      <td>0.803581</td>\n",
              "      <td>0.803097</td>\n",
              "      <td>0.805333</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>14</td>\n",
              "      <td>0.452400</td>\n",
              "      <td>0.484163</td>\n",
              "      <td>0.806667</td>\n",
              "      <td>0.805693</td>\n",
              "      <td>0.805061</td>\n",
              "      <td>0.806667</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table><p>"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              "\n",
              "    <div>\n",
              "      \n",
              "      <progress value='24' max='24' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
              "      [24/24 00:07]\n",
              "    </div>\n",
              "    "
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "   eval_loss  eval_accuracy   eval_f1  eval_precision  eval_recall  eval_runtime  \\\n",
              "0    0.47919       0.806333  0.805723         0.80541     0.806333        7.4079   \n",
              "\n",
              "   eval_samples_per_second  eval_steps_per_second  epoch  \n",
              "0                   404.97                   3.24   14.0  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-44237602-4bf9-4c36-b2f8-c931ee455d19\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>eval_loss</th>\n",
              "      <th>eval_accuracy</th>\n",
              "      <th>eval_f1</th>\n",
              "      <th>eval_precision</th>\n",
              "      <th>eval_recall</th>\n",
              "      <th>eval_runtime</th>\n",
              "      <th>eval_samples_per_second</th>\n",
              "      <th>eval_steps_per_second</th>\n",
              "      <th>epoch</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>0.47919</td>\n",
              "      <td>0.806333</td>\n",
              "      <td>0.805723</td>\n",
              "      <td>0.80541</td>\n",
              "      <td>0.806333</td>\n",
              "      <td>7.4079</td>\n",
              "      <td>404.97</td>\n",
              "      <td>3.24</td>\n",
              "      <td>14.0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-44237602-4bf9-4c36-b2f8-c931ee455d19')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-44237602-4bf9-4c36-b2f8-c931ee455d19 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-44237602-4bf9-4c36-b2f8-c931ee455d19');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "model = CustomModelLastHiddenLayer(Config,\n",
        "                                   requires_grad_layer=True, requires_grad_pooler=True,\n",
        "                                   requires_grad_layer_slit=-2)\n",
        "training_args, save_path = set_args(Config)\n",
        "\n",
        "trainer = Trainer(model=model, args=training_args,\n",
        "                  compute_metrics=compute_metrics,\n",
        "                  train_dataset=trn_dataset,\n",
        "                  eval_dataset=val_dataset,\n",
        "                  tokenizer=tokenizer,\n",
        "                  callbacks=[EarlyStoppingCallback(\n",
        "                      early_stopping_patience=Config.early_stopping_patience\n",
        "                      )],\n",
        "                  data_collator=DataCollatorWithPadding(\n",
        "                      tokenizer=tokenizer, padding='longest',\n",
        "                      ),\n",
        "                  )\n",
        "\n",
        "trn_result = trainer.train()\n",
        "val_result = trainer.evaluate(eval_dataset=val_dataset)\n",
        "display(pd.DataFrame(val_result, index=[0]))\n",
        "\n",
        "trainer.save_model(save_path)\n",
        "save_path_2 = save_path\n",
        "\n",
        "torch.cuda.empty_cache(); gc.collect();"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "id": "j8RZRpJInSJv",
        "outputId": "c4cd599d-2ac0-4238-a624-e48648e6ca6b"
      },
      "execution_count": 17,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "********************\n",
            "model name: jarvisx17/japanese-sentiment-analysis, device: cuda\n",
            "model hidden_size: 768, num_hidden_layers: 12\n",
            "=> num_classes: 3\n",
            "=> requires_grad is True list\n",
            "   encoder.layer.10.attention.self.query.weight\n",
            "   encoder.layer.10.attention.self.query.bias\n",
            "   encoder.layer.10.attention.self.key.weight\n",
            "   encoder.layer.10.attention.self.key.bias\n",
            "   encoder.layer.10.attention.self.value.weight\n",
            "   encoder.layer.10.attention.self.value.bias\n",
            "   encoder.layer.10.attention.output.dense.weight\n",
            "   encoder.layer.10.attention.output.dense.bias\n",
            "   encoder.layer.10.attention.output.LayerNorm.weight\n",
            "   encoder.layer.10.attention.output.LayerNorm.bias\n",
            "   encoder.layer.10.intermediate.dense.weight\n",
            "   encoder.layer.10.intermediate.dense.bias\n",
            "   encoder.layer.10.output.dense.weight\n",
            "   encoder.layer.10.output.dense.bias\n",
            "   encoder.layer.10.output.LayerNorm.weight\n",
            "   encoder.layer.10.output.LayerNorm.bias\n",
            "   encoder.layer.11.attention.self.query.weight\n",
            "   encoder.layer.11.attention.self.query.bias\n",
            "   encoder.layer.11.attention.self.key.weight\n",
            "   encoder.layer.11.attention.self.key.bias\n",
            "   encoder.layer.11.attention.self.value.weight\n",
            "   encoder.layer.11.attention.self.value.bias\n",
            "   encoder.layer.11.attention.output.dense.weight\n",
            "   encoder.layer.11.attention.output.dense.bias\n",
            "   encoder.layer.11.attention.output.LayerNorm.weight\n",
            "   encoder.layer.11.attention.output.LayerNorm.bias\n",
            "   encoder.layer.11.intermediate.dense.weight\n",
            "   encoder.layer.11.intermediate.dense.bias\n",
            "   encoder.layer.11.output.dense.weight\n",
            "   encoder.layer.11.output.dense.bias\n",
            "   encoder.layer.11.output.LayerNorm.weight\n",
            "   encoder.layer.11.output.LayerNorm.bias\n",
            "   pooler.dense.weight\n",
            "   pooler.dense.bias\n",
            "********************\n",
            "SET output_dir: 202349-22-24-45-jarvisx17/japanese-sentiment-analysis-finetuned-emotion\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              "\n",
              "    <div>\n",
              "      \n",
              "      <progress value='1343' max='2370' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
              "      [1343/2370 11:52 < 09:05, 1.88 it/s, Epoch 17/30]\n",
              "    </div>\n",
              "    <table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              " <tr style=\"text-align: left;\">\n",
              "      <th>Epoch</th>\n",
              "      <th>Training Loss</th>\n",
              "      <th>Validation Loss</th>\n",
              "      <th>Accuracy</th>\n",
              "      <th>F1</th>\n",
              "      <th>Precision</th>\n",
              "      <th>Recall</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <td>1</td>\n",
              "      <td>1.386900</td>\n",
              "      <td>0.908020</td>\n",
              "      <td>0.694333</td>\n",
              "      <td>0.667310</td>\n",
              "      <td>0.689318</td>\n",
              "      <td>0.694333</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2</td>\n",
              "      <td>0.696100</td>\n",
              "      <td>0.579840</td>\n",
              "      <td>0.745333</td>\n",
              "      <td>0.729889</td>\n",
              "      <td>0.737126</td>\n",
              "      <td>0.745333</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3</td>\n",
              "      <td>0.590300</td>\n",
              "      <td>0.548411</td>\n",
              "      <td>0.761667</td>\n",
              "      <td>0.753151</td>\n",
              "      <td>0.755059</td>\n",
              "      <td>0.761667</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>4</td>\n",
              "      <td>0.561500</td>\n",
              "      <td>0.537364</td>\n",
              "      <td>0.777333</td>\n",
              "      <td>0.767793</td>\n",
              "      <td>0.772806</td>\n",
              "      <td>0.777333</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>5</td>\n",
              "      <td>0.546100</td>\n",
              "      <td>0.510351</td>\n",
              "      <td>0.787000</td>\n",
              "      <td>0.779764</td>\n",
              "      <td>0.782623</td>\n",
              "      <td>0.787000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>6</td>\n",
              "      <td>0.527500</td>\n",
              "      <td>0.494195</td>\n",
              "      <td>0.791000</td>\n",
              "      <td>0.785818</td>\n",
              "      <td>0.786612</td>\n",
              "      <td>0.791000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>7</td>\n",
              "      <td>0.523100</td>\n",
              "      <td>0.489676</td>\n",
              "      <td>0.794667</td>\n",
              "      <td>0.790497</td>\n",
              "      <td>0.790915</td>\n",
              "      <td>0.794667</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>8</td>\n",
              "      <td>0.507000</td>\n",
              "      <td>0.492244</td>\n",
              "      <td>0.796000</td>\n",
              "      <td>0.790073</td>\n",
              "      <td>0.791936</td>\n",
              "      <td>0.796000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>9</td>\n",
              "      <td>0.499900</td>\n",
              "      <td>0.480021</td>\n",
              "      <td>0.800000</td>\n",
              "      <td>0.796319</td>\n",
              "      <td>0.796218</td>\n",
              "      <td>0.800000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>10</td>\n",
              "      <td>0.492400</td>\n",
              "      <td>0.495486</td>\n",
              "      <td>0.797333</td>\n",
              "      <td>0.794509</td>\n",
              "      <td>0.794917</td>\n",
              "      <td>0.797333</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>11</td>\n",
              "      <td>0.477700</td>\n",
              "      <td>0.482581</td>\n",
              "      <td>0.802667</td>\n",
              "      <td>0.799996</td>\n",
              "      <td>0.799792</td>\n",
              "      <td>0.802667</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>12</td>\n",
              "      <td>0.473000</td>\n",
              "      <td>0.496143</td>\n",
              "      <td>0.800667</td>\n",
              "      <td>0.793237</td>\n",
              "      <td>0.798471</td>\n",
              "      <td>0.800667</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>13</td>\n",
              "      <td>0.465200</td>\n",
              "      <td>0.480067</td>\n",
              "      <td>0.801667</td>\n",
              "      <td>0.798453</td>\n",
              "      <td>0.798282</td>\n",
              "      <td>0.801667</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>14</td>\n",
              "      <td>0.453900</td>\n",
              "      <td>0.480015</td>\n",
              "      <td>0.810333</td>\n",
              "      <td>0.808658</td>\n",
              "      <td>0.808009</td>\n",
              "      <td>0.810333</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>15</td>\n",
              "      <td>0.444000</td>\n",
              "      <td>0.493876</td>\n",
              "      <td>0.800000</td>\n",
              "      <td>0.795657</td>\n",
              "      <td>0.796175</td>\n",
              "      <td>0.800000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>16</td>\n",
              "      <td>0.438300</td>\n",
              "      <td>0.493622</td>\n",
              "      <td>0.801333</td>\n",
              "      <td>0.798256</td>\n",
              "      <td>0.798256</td>\n",
              "      <td>0.801333</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>17</td>\n",
              "      <td>0.428900</td>\n",
              "      <td>0.496532</td>\n",
              "      <td>0.802667</td>\n",
              "      <td>0.798797</td>\n",
              "      <td>0.799085</td>\n",
              "      <td>0.802667</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table><p>"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              "\n",
              "    <div>\n",
              "      \n",
              "      <progress value='24' max='24' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
              "      [24/24 00:07]\n",
              "    </div>\n",
              "    "
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "   eval_loss  eval_accuracy   eval_f1  eval_precision  eval_recall  eval_runtime  \\\n",
              "0   0.480015       0.810333  0.808658        0.808009     0.810333        7.4034   \n",
              "\n",
              "   eval_samples_per_second  eval_steps_per_second  epoch  \n",
              "0                  405.218                  3.242   17.0  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-799d04c1-125f-4a21-b437-917fa4cb7d8b\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>eval_loss</th>\n",
              "      <th>eval_accuracy</th>\n",
              "      <th>eval_f1</th>\n",
              "      <th>eval_precision</th>\n",
              "      <th>eval_recall</th>\n",
              "      <th>eval_runtime</th>\n",
              "      <th>eval_samples_per_second</th>\n",
              "      <th>eval_steps_per_second</th>\n",
              "      <th>epoch</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>0.480015</td>\n",
              "      <td>0.810333</td>\n",
              "      <td>0.808658</td>\n",
              "      <td>0.808009</td>\n",
              "      <td>0.810333</td>\n",
              "      <td>7.4034</td>\n",
              "      <td>405.218</td>\n",
              "      <td>3.242</td>\n",
              "      <td>17.0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-799d04c1-125f-4a21-b437-917fa4cb7d8b')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-799d04c1-125f-4a21-b437-917fa4cb7d8b button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-799d04c1-125f-4a21-b437-917fa4cb7d8b');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "model = CustomModelMultiHiddenLayer(Config, use_multi_layer=-4,\n",
        "                                    requires_grad_layer=True, requires_grad_pooler=True,\n",
        "                                    requires_grad_layer_slit=-2)\n",
        "training_args, save_path = set_args(Config)\n",
        "\n",
        "trainer = Trainer(model=model, args=training_args,\n",
        "                  compute_metrics=compute_metrics,\n",
        "                  train_dataset=trn_dataset,\n",
        "                  eval_dataset=val_dataset,\n",
        "                  tokenizer=tokenizer,\n",
        "                  callbacks=[EarlyStoppingCallback(\n",
        "                      early_stopping_patience=Config.early_stopping_patience\n",
        "                      )],\n",
        "                  data_collator=DataCollatorWithPadding(\n",
        "                      tokenizer=tokenizer, padding='longest',\n",
        "                      ),\n",
        "                  )\n",
        "\n",
        "trn_result = trainer.train()\n",
        "val_result = trainer.evaluate(eval_dataset=val_dataset)\n",
        "display(pd.DataFrame(val_result, index=[0]))\n",
        "\n",
        "trainer.save_model(save_path)\n",
        "save_path_3 = save_path\n",
        "\n",
        "torch.cuda.empty_cache(); gc.collect();"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "id": "T6Fw8mzArAFe",
        "outputId": "0f4234fe-9f34-4ad6-d96d-5cb5f73790d9"
      },
      "execution_count": 18,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "********************\n",
            "model name: jarvisx17/japanese-sentiment-analysis, device: cuda\n",
            "model hidden_size: 768, num_hidden_layers: 12\n",
            "=> num_classes: 3\n",
            "=> requires_grad is True list\n",
            "   encoder.layer.10.attention.self.query.weight\n",
            "   encoder.layer.10.attention.self.query.bias\n",
            "   encoder.layer.10.attention.self.key.weight\n",
            "   encoder.layer.10.attention.self.key.bias\n",
            "   encoder.layer.10.attention.self.value.weight\n",
            "   encoder.layer.10.attention.self.value.bias\n",
            "   encoder.layer.10.attention.output.dense.weight\n",
            "   encoder.layer.10.attention.output.dense.bias\n",
            "   encoder.layer.10.attention.output.LayerNorm.weight\n",
            "   encoder.layer.10.attention.output.LayerNorm.bias\n",
            "   encoder.layer.10.intermediate.dense.weight\n",
            "   encoder.layer.10.intermediate.dense.bias\n",
            "   encoder.layer.10.output.dense.weight\n",
            "   encoder.layer.10.output.dense.bias\n",
            "   encoder.layer.10.output.LayerNorm.weight\n",
            "   encoder.layer.10.output.LayerNorm.bias\n",
            "   encoder.layer.11.attention.self.query.weight\n",
            "   encoder.layer.11.attention.self.query.bias\n",
            "   encoder.layer.11.attention.self.key.weight\n",
            "   encoder.layer.11.attention.self.key.bias\n",
            "   encoder.layer.11.attention.self.value.weight\n",
            "   encoder.layer.11.attention.self.value.bias\n",
            "   encoder.layer.11.attention.output.dense.weight\n",
            "   encoder.layer.11.attention.output.dense.bias\n",
            "   encoder.layer.11.attention.output.LayerNorm.weight\n",
            "   encoder.layer.11.attention.output.LayerNorm.bias\n",
            "   encoder.layer.11.intermediate.dense.weight\n",
            "   encoder.layer.11.intermediate.dense.bias\n",
            "   encoder.layer.11.output.dense.weight\n",
            "   encoder.layer.11.output.dense.bias\n",
            "   encoder.layer.11.output.LayerNorm.weight\n",
            "   encoder.layer.11.output.LayerNorm.bias\n",
            "   pooler.dense.weight\n",
            "   pooler.dense.bias\n",
            "********************\n",
            "SET output_dir: 202349-22-36-47-jarvisx17/japanese-sentiment-analysis-finetuned-emotion\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              "\n",
              "    <div>\n",
              "      \n",
              "      <progress value='1343' max='2370' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
              "      [1343/2370 11:57 < 09:09, 1.87 it/s, Epoch 17/30]\n",
              "    </div>\n",
              "    <table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              " <tr style=\"text-align: left;\">\n",
              "      <th>Epoch</th>\n",
              "      <th>Training Loss</th>\n",
              "      <th>Validation Loss</th>\n",
              "      <th>Accuracy</th>\n",
              "      <th>F1</th>\n",
              "      <th>Precision</th>\n",
              "      <th>Recall</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <td>1</td>\n",
              "      <td>1.020500</td>\n",
              "      <td>0.752971</td>\n",
              "      <td>0.713333</td>\n",
              "      <td>0.679213</td>\n",
              "      <td>0.708823</td>\n",
              "      <td>0.713333</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2</td>\n",
              "      <td>0.662000</td>\n",
              "      <td>0.583991</td>\n",
              "      <td>0.748333</td>\n",
              "      <td>0.730099</td>\n",
              "      <td>0.742794</td>\n",
              "      <td>0.748333</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3</td>\n",
              "      <td>0.595100</td>\n",
              "      <td>0.553902</td>\n",
              "      <td>0.766667</td>\n",
              "      <td>0.756462</td>\n",
              "      <td>0.760486</td>\n",
              "      <td>0.766667</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>4</td>\n",
              "      <td>0.565900</td>\n",
              "      <td>0.547183</td>\n",
              "      <td>0.773333</td>\n",
              "      <td>0.761913</td>\n",
              "      <td>0.769502</td>\n",
              "      <td>0.773333</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>5</td>\n",
              "      <td>0.553300</td>\n",
              "      <td>0.515359</td>\n",
              "      <td>0.784000</td>\n",
              "      <td>0.776270</td>\n",
              "      <td>0.779625</td>\n",
              "      <td>0.784000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>6</td>\n",
              "      <td>0.530700</td>\n",
              "      <td>0.497857</td>\n",
              "      <td>0.790333</td>\n",
              "      <td>0.784277</td>\n",
              "      <td>0.786037</td>\n",
              "      <td>0.790333</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>7</td>\n",
              "      <td>0.527300</td>\n",
              "      <td>0.493129</td>\n",
              "      <td>0.795333</td>\n",
              "      <td>0.790788</td>\n",
              "      <td>0.791440</td>\n",
              "      <td>0.795333</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>8</td>\n",
              "      <td>0.512000</td>\n",
              "      <td>0.496085</td>\n",
              "      <td>0.792000</td>\n",
              "      <td>0.785697</td>\n",
              "      <td>0.787868</td>\n",
              "      <td>0.792000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>9</td>\n",
              "      <td>0.505500</td>\n",
              "      <td>0.478333</td>\n",
              "      <td>0.800000</td>\n",
              "      <td>0.797489</td>\n",
              "      <td>0.796726</td>\n",
              "      <td>0.800000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>10</td>\n",
              "      <td>0.499000</td>\n",
              "      <td>0.487472</td>\n",
              "      <td>0.804333</td>\n",
              "      <td>0.802069</td>\n",
              "      <td>0.801984</td>\n",
              "      <td>0.804333</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>11</td>\n",
              "      <td>0.485000</td>\n",
              "      <td>0.477657</td>\n",
              "      <td>0.802667</td>\n",
              "      <td>0.801337</td>\n",
              "      <td>0.800855</td>\n",
              "      <td>0.802667</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>12</td>\n",
              "      <td>0.479200</td>\n",
              "      <td>0.493623</td>\n",
              "      <td>0.800333</td>\n",
              "      <td>0.793994</td>\n",
              "      <td>0.797169</td>\n",
              "      <td>0.800333</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>13</td>\n",
              "      <td>0.469400</td>\n",
              "      <td>0.474700</td>\n",
              "      <td>0.807667</td>\n",
              "      <td>0.806237</td>\n",
              "      <td>0.805640</td>\n",
              "      <td>0.807667</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>14</td>\n",
              "      <td>0.465100</td>\n",
              "      <td>0.478021</td>\n",
              "      <td>0.811667</td>\n",
              "      <td>0.810220</td>\n",
              "      <td>0.809533</td>\n",
              "      <td>0.811667</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>15</td>\n",
              "      <td>0.455500</td>\n",
              "      <td>0.488340</td>\n",
              "      <td>0.808333</td>\n",
              "      <td>0.806021</td>\n",
              "      <td>0.805731</td>\n",
              "      <td>0.808333</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>16</td>\n",
              "      <td>0.449200</td>\n",
              "      <td>0.484330</td>\n",
              "      <td>0.803667</td>\n",
              "      <td>0.802156</td>\n",
              "      <td>0.801609</td>\n",
              "      <td>0.803667</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>17</td>\n",
              "      <td>0.437800</td>\n",
              "      <td>0.498368</td>\n",
              "      <td>0.803000</td>\n",
              "      <td>0.798341</td>\n",
              "      <td>0.799339</td>\n",
              "      <td>0.803000</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table><p>"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              "\n",
              "    <div>\n",
              "      \n",
              "      <progress value='24' max='24' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
              "      [24/24 00:07]\n",
              "    </div>\n",
              "    "
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "   eval_loss  eval_accuracy  eval_f1  eval_precision  eval_recall  eval_runtime  \\\n",
              "0   0.478021       0.811667  0.81022        0.809533     0.811667         7.436   \n",
              "\n",
              "   eval_samples_per_second  eval_steps_per_second  epoch  \n",
              "0                  403.444                  3.228   17.0  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-1d4f92ce-e6e3-47bb-94b7-5406a8c40ef8\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>eval_loss</th>\n",
              "      <th>eval_accuracy</th>\n",
              "      <th>eval_f1</th>\n",
              "      <th>eval_precision</th>\n",
              "      <th>eval_recall</th>\n",
              "      <th>eval_runtime</th>\n",
              "      <th>eval_samples_per_second</th>\n",
              "      <th>eval_steps_per_second</th>\n",
              "      <th>epoch</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>0.478021</td>\n",
              "      <td>0.811667</td>\n",
              "      <td>0.81022</td>\n",
              "      <td>0.809533</td>\n",
              "      <td>0.811667</td>\n",
              "      <td>7.436</td>\n",
              "      <td>403.444</td>\n",
              "      <td>3.228</td>\n",
              "      <td>17.0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-1d4f92ce-e6e3-47bb-94b7-5406a8c40ef8')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-1d4f92ce-e6e3-47bb-94b7-5406a8c40ef8 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-1d4f92ce-e6e3-47bb-94b7-5406a8c40ef8');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "model = CustomModelLastHiddenLayerMeanPooling(Config,\n",
        "                                              requires_grad_layer=True, requires_grad_pooler=True,\n",
        "                                              requires_grad_layer_slit=-2)\n",
        "training_args, save_path = set_args(Config)\n",
        "\n",
        "trainer = Trainer(model=model, args=training_args,\n",
        "                  compute_metrics=compute_metrics,\n",
        "                  train_dataset=trn_dataset,\n",
        "                  eval_dataset=val_dataset,\n",
        "                  tokenizer=tokenizer,\n",
        "                  callbacks=[EarlyStoppingCallback(\n",
        "                      early_stopping_patience=Config.early_stopping_patience\n",
        "                      )],\n",
        "                  data_collator=DataCollatorWithPadding(\n",
        "                      tokenizer=tokenizer, padding='longest',\n",
        "                      ),\n",
        "                  )\n",
        "\n",
        "trn_result = trainer.train()\n",
        "val_result = trainer.evaluate(eval_dataset=val_dataset)\n",
        "display(pd.DataFrame(val_result, index=[0]))\n",
        "\n",
        "trainer.save_model(save_path)\n",
        "save_path_4 = save_path\n",
        "\n",
        "torch.cuda.empty_cache(); gc.collect();"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "id": "YFywCEitrCR2",
        "outputId": "757d5bdb-a247-4fe4-87fe-628f2ec6d3ba"
      },
      "execution_count": 19,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "********************\n",
            "model name: jarvisx17/japanese-sentiment-analysis, device: cuda\n",
            "model hidden_size: 768, num_hidden_layers: 12\n",
            "=> num_classes: 3\n",
            "=> requires_grad is True list\n",
            "   encoder.layer.10.attention.self.query.weight\n",
            "   encoder.layer.10.attention.self.query.bias\n",
            "   encoder.layer.10.attention.self.key.weight\n",
            "   encoder.layer.10.attention.self.key.bias\n",
            "   encoder.layer.10.attention.self.value.weight\n",
            "   encoder.layer.10.attention.self.value.bias\n",
            "   encoder.layer.10.attention.output.dense.weight\n",
            "   encoder.layer.10.attention.output.dense.bias\n",
            "   encoder.layer.10.attention.output.LayerNorm.weight\n",
            "   encoder.layer.10.attention.output.LayerNorm.bias\n",
            "   encoder.layer.10.intermediate.dense.weight\n",
            "   encoder.layer.10.intermediate.dense.bias\n",
            "   encoder.layer.10.output.dense.weight\n",
            "   encoder.layer.10.output.dense.bias\n",
            "   encoder.layer.10.output.LayerNorm.weight\n",
            "   encoder.layer.10.output.LayerNorm.bias\n",
            "   encoder.layer.11.attention.self.query.weight\n",
            "   encoder.layer.11.attention.self.query.bias\n",
            "   encoder.layer.11.attention.self.key.weight\n",
            "   encoder.layer.11.attention.self.key.bias\n",
            "   encoder.layer.11.attention.self.value.weight\n",
            "   encoder.layer.11.attention.self.value.bias\n",
            "   encoder.layer.11.attention.output.dense.weight\n",
            "   encoder.layer.11.attention.output.dense.bias\n",
            "   encoder.layer.11.attention.output.LayerNorm.weight\n",
            "   encoder.layer.11.attention.output.LayerNorm.bias\n",
            "   encoder.layer.11.intermediate.dense.weight\n",
            "   encoder.layer.11.intermediate.dense.bias\n",
            "   encoder.layer.11.output.dense.weight\n",
            "   encoder.layer.11.output.dense.bias\n",
            "   encoder.layer.11.output.LayerNorm.weight\n",
            "   encoder.layer.11.output.LayerNorm.bias\n",
            "   pooler.dense.weight\n",
            "   pooler.dense.bias\n",
            "********************\n",
            "SET output_dir: 202349-22-48-55-jarvisx17/japanese-sentiment-analysis-finetuned-emotion\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              "\n",
              "    <div>\n",
              "      \n",
              "      <progress value='1106' max='2370' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
              "      [1106/2370 09:47 < 11:12, 1.88 it/s, Epoch 14/30]\n",
              "    </div>\n",
              "    <table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              " <tr style=\"text-align: left;\">\n",
              "      <th>Epoch</th>\n",
              "      <th>Training Loss</th>\n",
              "      <th>Validation Loss</th>\n",
              "      <th>Accuracy</th>\n",
              "      <th>F1</th>\n",
              "      <th>Precision</th>\n",
              "      <th>Recall</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <td>1</td>\n",
              "      <td>1.208200</td>\n",
              "      <td>0.846504</td>\n",
              "      <td>0.714000</td>\n",
              "      <td>0.683148</td>\n",
              "      <td>0.708157</td>\n",
              "      <td>0.714000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2</td>\n",
              "      <td>0.670300</td>\n",
              "      <td>0.565059</td>\n",
              "      <td>0.754667</td>\n",
              "      <td>0.740097</td>\n",
              "      <td>0.748683</td>\n",
              "      <td>0.754667</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3</td>\n",
              "      <td>0.576300</td>\n",
              "      <td>0.530186</td>\n",
              "      <td>0.776667</td>\n",
              "      <td>0.768999</td>\n",
              "      <td>0.771022</td>\n",
              "      <td>0.776667</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>4</td>\n",
              "      <td>0.543100</td>\n",
              "      <td>0.522558</td>\n",
              "      <td>0.782667</td>\n",
              "      <td>0.774631</td>\n",
              "      <td>0.777958</td>\n",
              "      <td>0.782667</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>5</td>\n",
              "      <td>0.532100</td>\n",
              "      <td>0.501566</td>\n",
              "      <td>0.790000</td>\n",
              "      <td>0.783734</td>\n",
              "      <td>0.785616</td>\n",
              "      <td>0.790000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>6</td>\n",
              "      <td>0.520100</td>\n",
              "      <td>0.488030</td>\n",
              "      <td>0.796667</td>\n",
              "      <td>0.793393</td>\n",
              "      <td>0.792952</td>\n",
              "      <td>0.796667</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>7</td>\n",
              "      <td>0.513000</td>\n",
              "      <td>0.488709</td>\n",
              "      <td>0.793667</td>\n",
              "      <td>0.789214</td>\n",
              "      <td>0.789784</td>\n",
              "      <td>0.793667</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>8</td>\n",
              "      <td>0.495700</td>\n",
              "      <td>0.495782</td>\n",
              "      <td>0.794667</td>\n",
              "      <td>0.788791</td>\n",
              "      <td>0.790969</td>\n",
              "      <td>0.794667</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>9</td>\n",
              "      <td>0.487400</td>\n",
              "      <td>0.484047</td>\n",
              "      <td>0.798667</td>\n",
              "      <td>0.795484</td>\n",
              "      <td>0.795217</td>\n",
              "      <td>0.798667</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>10</td>\n",
              "      <td>0.480800</td>\n",
              "      <td>0.501845</td>\n",
              "      <td>0.797000</td>\n",
              "      <td>0.793585</td>\n",
              "      <td>0.793885</td>\n",
              "      <td>0.797000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>11</td>\n",
              "      <td>0.466500</td>\n",
              "      <td>0.476218</td>\n",
              "      <td>0.802000</td>\n",
              "      <td>0.801521</td>\n",
              "      <td>0.801229</td>\n",
              "      <td>0.802000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>12</td>\n",
              "      <td>0.456000</td>\n",
              "      <td>0.507635</td>\n",
              "      <td>0.795333</td>\n",
              "      <td>0.787235</td>\n",
              "      <td>0.793448</td>\n",
              "      <td>0.795333</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>13</td>\n",
              "      <td>0.450000</td>\n",
              "      <td>0.479758</td>\n",
              "      <td>0.802000</td>\n",
              "      <td>0.800880</td>\n",
              "      <td>0.800185</td>\n",
              "      <td>0.802000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>14</td>\n",
              "      <td>0.434800</td>\n",
              "      <td>0.499136</td>\n",
              "      <td>0.801667</td>\n",
              "      <td>0.796642</td>\n",
              "      <td>0.799110</td>\n",
              "      <td>0.801667</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table><p>"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              "\n",
              "    <div>\n",
              "      \n",
              "      <progress value='24' max='24' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
              "      [24/24 00:07]\n",
              "    </div>\n",
              "    "
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "   eval_loss  eval_accuracy   eval_f1  eval_precision  eval_recall  eval_runtime  \\\n",
              "0   0.476218          0.802  0.801521        0.801229        0.802        7.4628   \n",
              "\n",
              "   eval_samples_per_second  eval_steps_per_second  epoch  \n",
              "0                  401.995                  3.216   14.0  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-69b62af1-3137-454a-8f74-d30da484efe1\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>eval_loss</th>\n",
              "      <th>eval_accuracy</th>\n",
              "      <th>eval_f1</th>\n",
              "      <th>eval_precision</th>\n",
              "      <th>eval_recall</th>\n",
              "      <th>eval_runtime</th>\n",
              "      <th>eval_samples_per_second</th>\n",
              "      <th>eval_steps_per_second</th>\n",
              "      <th>epoch</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>0.476218</td>\n",
              "      <td>0.802</td>\n",
              "      <td>0.801521</td>\n",
              "      <td>0.801229</td>\n",
              "      <td>0.802</td>\n",
              "      <td>7.4628</td>\n",
              "      <td>401.995</td>\n",
              "      <td>3.216</td>\n",
              "      <td>14.0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-69b62af1-3137-454a-8f74-d30da484efe1')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-69b62af1-3137-454a-8f74-d30da484efe1 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-69b62af1-3137-454a-8f74-d30da484efe1');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "model = CustomModelLastHiddenLayerMaxPooling(Config,\n",
        "                                             requires_grad_layer=True, requires_grad_pooler=True,\n",
        "                                             requires_grad_layer_slit=-2)\n",
        "training_args, save_path = set_args(Config)\n",
        "\n",
        "trainer = Trainer(model=model, args=training_args,\n",
        "                  compute_metrics=compute_metrics,\n",
        "                  train_dataset=trn_dataset,\n",
        "                  eval_dataset=val_dataset,\n",
        "                  tokenizer=tokenizer,\n",
        "                  callbacks=[EarlyStoppingCallback(\n",
        "                      early_stopping_patience=Config.early_stopping_patience\n",
        "                      )],\n",
        "                  data_collator=DataCollatorWithPadding(\n",
        "                      tokenizer=tokenizer, padding='longest',\n",
        "                      ),\n",
        "                  )\n",
        "\n",
        "trn_result = trainer.train()\n",
        "val_result = trainer.evaluate(eval_dataset=val_dataset)\n",
        "display(pd.DataFrame(val_result, index=[0]))\n",
        "\n",
        "trainer.save_model(save_path)\n",
        "save_path_5 = save_path\n",
        "\n",
        "torch.cuda.empty_cache(); gc.collect();"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "id": "i9nEBcYnrE-b",
        "outputId": "0cec7a89-70a2-416c-f95d-7b0425955ae3"
      },
      "execution_count": 20,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "********************\n",
            "model name: jarvisx17/japanese-sentiment-analysis, device: cuda\n",
            "model hidden_size: 768, num_hidden_layers: 12\n",
            "=> num_classes: 3\n",
            "=> requires_grad is True list\n",
            "   encoder.layer.10.attention.self.query.weight\n",
            "   encoder.layer.10.attention.self.query.bias\n",
            "   encoder.layer.10.attention.self.key.weight\n",
            "   encoder.layer.10.attention.self.key.bias\n",
            "   encoder.layer.10.attention.self.value.weight\n",
            "   encoder.layer.10.attention.self.value.bias\n",
            "   encoder.layer.10.attention.output.dense.weight\n",
            "   encoder.layer.10.attention.output.dense.bias\n",
            "   encoder.layer.10.attention.output.LayerNorm.weight\n",
            "   encoder.layer.10.attention.output.LayerNorm.bias\n",
            "   encoder.layer.10.intermediate.dense.weight\n",
            "   encoder.layer.10.intermediate.dense.bias\n",
            "   encoder.layer.10.output.dense.weight\n",
            "   encoder.layer.10.output.dense.bias\n",
            "   encoder.layer.10.output.LayerNorm.weight\n",
            "   encoder.layer.10.output.LayerNorm.bias\n",
            "   encoder.layer.11.attention.self.query.weight\n",
            "   encoder.layer.11.attention.self.query.bias\n",
            "   encoder.layer.11.attention.self.key.weight\n",
            "   encoder.layer.11.attention.self.key.bias\n",
            "   encoder.layer.11.attention.self.value.weight\n",
            "   encoder.layer.11.attention.self.value.bias\n",
            "   encoder.layer.11.attention.output.dense.weight\n",
            "   encoder.layer.11.attention.output.dense.bias\n",
            "   encoder.layer.11.attention.output.LayerNorm.weight\n",
            "   encoder.layer.11.attention.output.LayerNorm.bias\n",
            "   encoder.layer.11.intermediate.dense.weight\n",
            "   encoder.layer.11.intermediate.dense.bias\n",
            "   encoder.layer.11.output.dense.weight\n",
            "   encoder.layer.11.output.dense.bias\n",
            "   encoder.layer.11.output.LayerNorm.weight\n",
            "   encoder.layer.11.output.LayerNorm.bias\n",
            "   pooler.dense.weight\n",
            "   pooler.dense.bias\n",
            "********************\n",
            "SET output_dir: 202349-22-58-52-jarvisx17/japanese-sentiment-analysis-finetuned-emotion\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              "\n",
              "    <div>\n",
              "      \n",
              "      <progress value='948' max='2370' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
              "      [ 948/2370 08:23 < 12:36, 1.88 it/s, Epoch 12/30]\n",
              "    </div>\n",
              "    <table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              " <tr style=\"text-align: left;\">\n",
              "      <th>Epoch</th>\n",
              "      <th>Training Loss</th>\n",
              "      <th>Validation Loss</th>\n",
              "      <th>Accuracy</th>\n",
              "      <th>F1</th>\n",
              "      <th>Precision</th>\n",
              "      <th>Recall</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <td>1</td>\n",
              "      <td>1.268300</td>\n",
              "      <td>0.936162</td>\n",
              "      <td>0.650333</td>\n",
              "      <td>0.636977</td>\n",
              "      <td>0.641610</td>\n",
              "      <td>0.650333</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2</td>\n",
              "      <td>0.733300</td>\n",
              "      <td>0.601542</td>\n",
              "      <td>0.731000</td>\n",
              "      <td>0.708197</td>\n",
              "      <td>0.724717</td>\n",
              "      <td>0.731000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3</td>\n",
              "      <td>0.613900</td>\n",
              "      <td>0.560194</td>\n",
              "      <td>0.757667</td>\n",
              "      <td>0.742857</td>\n",
              "      <td>0.752795</td>\n",
              "      <td>0.757667</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>4</td>\n",
              "      <td>0.571400</td>\n",
              "      <td>0.540149</td>\n",
              "      <td>0.771000</td>\n",
              "      <td>0.759356</td>\n",
              "      <td>0.766618</td>\n",
              "      <td>0.771000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>5</td>\n",
              "      <td>0.551800</td>\n",
              "      <td>0.516131</td>\n",
              "      <td>0.781000</td>\n",
              "      <td>0.773057</td>\n",
              "      <td>0.776054</td>\n",
              "      <td>0.781000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>6</td>\n",
              "      <td>0.535300</td>\n",
              "      <td>0.503443</td>\n",
              "      <td>0.787667</td>\n",
              "      <td>0.781459</td>\n",
              "      <td>0.783193</td>\n",
              "      <td>0.787667</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>7</td>\n",
              "      <td>0.530600</td>\n",
              "      <td>0.498205</td>\n",
              "      <td>0.787000</td>\n",
              "      <td>0.781409</td>\n",
              "      <td>0.782600</td>\n",
              "      <td>0.787000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>8</td>\n",
              "      <td>0.510200</td>\n",
              "      <td>0.501660</td>\n",
              "      <td>0.788667</td>\n",
              "      <td>0.780333</td>\n",
              "      <td>0.785557</td>\n",
              "      <td>0.788667</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>9</td>\n",
              "      <td>0.500700</td>\n",
              "      <td>0.483214</td>\n",
              "      <td>0.799667</td>\n",
              "      <td>0.796819</td>\n",
              "      <td>0.796331</td>\n",
              "      <td>0.799667</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>10</td>\n",
              "      <td>0.498000</td>\n",
              "      <td>0.490943</td>\n",
              "      <td>0.796667</td>\n",
              "      <td>0.793659</td>\n",
              "      <td>0.793401</td>\n",
              "      <td>0.796667</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>11</td>\n",
              "      <td>0.481500</td>\n",
              "      <td>0.487638</td>\n",
              "      <td>0.801333</td>\n",
              "      <td>0.796499</td>\n",
              "      <td>0.797763</td>\n",
              "      <td>0.801333</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>12</td>\n",
              "      <td>0.470800</td>\n",
              "      <td>0.489562</td>\n",
              "      <td>0.796000</td>\n",
              "      <td>0.789784</td>\n",
              "      <td>0.792753</td>\n",
              "      <td>0.796000</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table><p>"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              "\n",
              "    <div>\n",
              "      \n",
              "      <progress value='24' max='24' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
              "      [24/24 00:07]\n",
              "    </div>\n",
              "    "
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "   eval_loss  eval_accuracy   eval_f1  eval_precision  eval_recall  eval_runtime  \\\n",
              "0   0.483214       0.799667  0.796819        0.796331     0.799667        7.4174   \n",
              "\n",
              "   eval_samples_per_second  eval_steps_per_second  epoch  \n",
              "0                  404.457                  3.236   12.0  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-8c87461a-0bf5-4005-a531-7e0634a727c4\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>eval_loss</th>\n",
              "      <th>eval_accuracy</th>\n",
              "      <th>eval_f1</th>\n",
              "      <th>eval_precision</th>\n",
              "      <th>eval_recall</th>\n",
              "      <th>eval_runtime</th>\n",
              "      <th>eval_samples_per_second</th>\n",
              "      <th>eval_steps_per_second</th>\n",
              "      <th>epoch</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>0.483214</td>\n",
              "      <td>0.799667</td>\n",
              "      <td>0.796819</td>\n",
              "      <td>0.796331</td>\n",
              "      <td>0.799667</td>\n",
              "      <td>7.4174</td>\n",
              "      <td>404.457</td>\n",
              "      <td>3.236</td>\n",
              "      <td>12.0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-8c87461a-0bf5-4005-a531-7e0634a727c4')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-8c87461a-0bf5-4005-a531-7e0634a727c4 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-8c87461a-0bf5-4005-a531-7e0634a727c4');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "model = CustomModelLastHiddenLayerMeanMaxPooling(Config,\n",
        "                                                 requires_grad_layer=True, requires_grad_pooler=True,\n",
        "                                                 requires_grad_layer_slit=-2)\n",
        "training_args, save_path = set_args(Config)\n",
        "\n",
        "trainer = Trainer(model=model, args=training_args,\n",
        "                  compute_metrics=compute_metrics,\n",
        "                  train_dataset=trn_dataset,\n",
        "                  eval_dataset=val_dataset,\n",
        "                  tokenizer=tokenizer,\n",
        "                  callbacks=[EarlyStoppingCallback(\n",
        "                      early_stopping_patience=Config.early_stopping_patience\n",
        "                      )],\n",
        "                  data_collator=DataCollatorWithPadding(\n",
        "                      tokenizer=tokenizer, padding='longest',\n",
        "                      ),\n",
        "                  )\n",
        "\n",
        "trn_result = trainer.train()\n",
        "val_result = trainer.evaluate(eval_dataset=val_dataset)\n",
        "display(pd.DataFrame(val_result, index=[0]))\n",
        "\n",
        "trainer.save_model(save_path)\n",
        "save_path_6 = save_path\n",
        "\n",
        "torch.cuda.empty_cache(); gc.collect();"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "id": "TcYp2BX_nC68",
        "outputId": "95a4e67d-6a70-40cd-976f-5db36be5ab56"
      },
      "execution_count": 21,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "********************\n",
            "model name: jarvisx17/japanese-sentiment-analysis, device: cuda\n",
            "model hidden_size: 768, num_hidden_layers: 12\n",
            "=> num_classes: 3\n",
            "=> requires_grad is True list\n",
            "   encoder.layer.10.attention.self.query.weight\n",
            "   encoder.layer.10.attention.self.query.bias\n",
            "   encoder.layer.10.attention.self.key.weight\n",
            "   encoder.layer.10.attention.self.key.bias\n",
            "   encoder.layer.10.attention.self.value.weight\n",
            "   encoder.layer.10.attention.self.value.bias\n",
            "   encoder.layer.10.attention.output.dense.weight\n",
            "   encoder.layer.10.attention.output.dense.bias\n",
            "   encoder.layer.10.attention.output.LayerNorm.weight\n",
            "   encoder.layer.10.attention.output.LayerNorm.bias\n",
            "   encoder.layer.10.intermediate.dense.weight\n",
            "   encoder.layer.10.intermediate.dense.bias\n",
            "   encoder.layer.10.output.dense.weight\n",
            "   encoder.layer.10.output.dense.bias\n",
            "   encoder.layer.10.output.LayerNorm.weight\n",
            "   encoder.layer.10.output.LayerNorm.bias\n",
            "   encoder.layer.11.attention.self.query.weight\n",
            "   encoder.layer.11.attention.self.query.bias\n",
            "   encoder.layer.11.attention.self.key.weight\n",
            "   encoder.layer.11.attention.self.key.bias\n",
            "   encoder.layer.11.attention.self.value.weight\n",
            "   encoder.layer.11.attention.self.value.bias\n",
            "   encoder.layer.11.attention.output.dense.weight\n",
            "   encoder.layer.11.attention.output.dense.bias\n",
            "   encoder.layer.11.attention.output.LayerNorm.weight\n",
            "   encoder.layer.11.attention.output.LayerNorm.bias\n",
            "   encoder.layer.11.intermediate.dense.weight\n",
            "   encoder.layer.11.intermediate.dense.bias\n",
            "   encoder.layer.11.output.dense.weight\n",
            "   encoder.layer.11.output.dense.bias\n",
            "   encoder.layer.11.output.LayerNorm.weight\n",
            "   encoder.layer.11.output.LayerNorm.bias\n",
            "   pooler.dense.weight\n",
            "   pooler.dense.bias\n",
            "********************\n",
            "SET output_dir: 202349-23-7-25-jarvisx17/japanese-sentiment-analysis-finetuned-emotion\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              "\n",
              "    <div>\n",
              "      \n",
              "      <progress value='1817' max='2370' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
              "      [1817/2370 16:07 < 04:54, 1.88 it/s, Epoch 23/30]\n",
              "    </div>\n",
              "    <table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              " <tr style=\"text-align: left;\">\n",
              "      <th>Epoch</th>\n",
              "      <th>Training Loss</th>\n",
              "      <th>Validation Loss</th>\n",
              "      <th>Accuracy</th>\n",
              "      <th>F1</th>\n",
              "      <th>Precision</th>\n",
              "      <th>Recall</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <td>1</td>\n",
              "      <td>1.131400</td>\n",
              "      <td>0.837980</td>\n",
              "      <td>0.649333</td>\n",
              "      <td>0.535221</td>\n",
              "      <td>0.664287</td>\n",
              "      <td>0.649333</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2</td>\n",
              "      <td>0.687100</td>\n",
              "      <td>0.575958</td>\n",
              "      <td>0.750000</td>\n",
              "      <td>0.733245</td>\n",
              "      <td>0.744749</td>\n",
              "      <td>0.750000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3</td>\n",
              "      <td>0.595500</td>\n",
              "      <td>0.541353</td>\n",
              "      <td>0.771333</td>\n",
              "      <td>0.763620</td>\n",
              "      <td>0.765418</td>\n",
              "      <td>0.771333</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>4</td>\n",
              "      <td>0.559500</td>\n",
              "      <td>0.530109</td>\n",
              "      <td>0.785000</td>\n",
              "      <td>0.776529</td>\n",
              "      <td>0.780903</td>\n",
              "      <td>0.785000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>5</td>\n",
              "      <td>0.545500</td>\n",
              "      <td>0.505849</td>\n",
              "      <td>0.792667</td>\n",
              "      <td>0.787427</td>\n",
              "      <td>0.788368</td>\n",
              "      <td>0.792667</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>6</td>\n",
              "      <td>0.531000</td>\n",
              "      <td>0.494763</td>\n",
              "      <td>0.795667</td>\n",
              "      <td>0.791432</td>\n",
              "      <td>0.791608</td>\n",
              "      <td>0.795667</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>7</td>\n",
              "      <td>0.523800</td>\n",
              "      <td>0.493043</td>\n",
              "      <td>0.793333</td>\n",
              "      <td>0.789836</td>\n",
              "      <td>0.789785</td>\n",
              "      <td>0.793333</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>8</td>\n",
              "      <td>0.507800</td>\n",
              "      <td>0.494541</td>\n",
              "      <td>0.795333</td>\n",
              "      <td>0.789840</td>\n",
              "      <td>0.791482</td>\n",
              "      <td>0.795333</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>9</td>\n",
              "      <td>0.499800</td>\n",
              "      <td>0.481815</td>\n",
              "      <td>0.797000</td>\n",
              "      <td>0.794776</td>\n",
              "      <td>0.793984</td>\n",
              "      <td>0.797000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>10</td>\n",
              "      <td>0.496100</td>\n",
              "      <td>0.495787</td>\n",
              "      <td>0.794333</td>\n",
              "      <td>0.792800</td>\n",
              "      <td>0.793278</td>\n",
              "      <td>0.794333</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>11</td>\n",
              "      <td>0.480600</td>\n",
              "      <td>0.481336</td>\n",
              "      <td>0.799333</td>\n",
              "      <td>0.797131</td>\n",
              "      <td>0.796449</td>\n",
              "      <td>0.799333</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>12</td>\n",
              "      <td>0.470200</td>\n",
              "      <td>0.488128</td>\n",
              "      <td>0.802333</td>\n",
              "      <td>0.797152</td>\n",
              "      <td>0.798971</td>\n",
              "      <td>0.802333</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>13</td>\n",
              "      <td>0.467000</td>\n",
              "      <td>0.478546</td>\n",
              "      <td>0.801333</td>\n",
              "      <td>0.799339</td>\n",
              "      <td>0.798654</td>\n",
              "      <td>0.801333</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>14</td>\n",
              "      <td>0.460100</td>\n",
              "      <td>0.480080</td>\n",
              "      <td>0.804333</td>\n",
              "      <td>0.801819</td>\n",
              "      <td>0.801264</td>\n",
              "      <td>0.804333</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>15</td>\n",
              "      <td>0.447200</td>\n",
              "      <td>0.487528</td>\n",
              "      <td>0.800333</td>\n",
              "      <td>0.796921</td>\n",
              "      <td>0.796845</td>\n",
              "      <td>0.800333</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>16</td>\n",
              "      <td>0.442300</td>\n",
              "      <td>0.484742</td>\n",
              "      <td>0.802667</td>\n",
              "      <td>0.799405</td>\n",
              "      <td>0.799491</td>\n",
              "      <td>0.802667</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>17</td>\n",
              "      <td>0.437300</td>\n",
              "      <td>0.491377</td>\n",
              "      <td>0.806667</td>\n",
              "      <td>0.802592</td>\n",
              "      <td>0.803322</td>\n",
              "      <td>0.806667</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>18</td>\n",
              "      <td>0.422500</td>\n",
              "      <td>0.481978</td>\n",
              "      <td>0.800333</td>\n",
              "      <td>0.797962</td>\n",
              "      <td>0.797494</td>\n",
              "      <td>0.800333</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>19</td>\n",
              "      <td>0.415600</td>\n",
              "      <td>0.489726</td>\n",
              "      <td>0.802667</td>\n",
              "      <td>0.798988</td>\n",
              "      <td>0.799405</td>\n",
              "      <td>0.802667</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>20</td>\n",
              "      <td>0.415200</td>\n",
              "      <td>0.491104</td>\n",
              "      <td>0.807000</td>\n",
              "      <td>0.804326</td>\n",
              "      <td>0.804050</td>\n",
              "      <td>0.807000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>21</td>\n",
              "      <td>0.405200</td>\n",
              "      <td>0.492123</td>\n",
              "      <td>0.805667</td>\n",
              "      <td>0.803532</td>\n",
              "      <td>0.803108</td>\n",
              "      <td>0.805667</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>22</td>\n",
              "      <td>0.405000</td>\n",
              "      <td>0.502021</td>\n",
              "      <td>0.802000</td>\n",
              "      <td>0.797106</td>\n",
              "      <td>0.798694</td>\n",
              "      <td>0.802000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>23</td>\n",
              "      <td>0.401000</td>\n",
              "      <td>0.504496</td>\n",
              "      <td>0.797000</td>\n",
              "      <td>0.792776</td>\n",
              "      <td>0.794100</td>\n",
              "      <td>0.797000</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table><p>"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              "\n",
              "    <div>\n",
              "      \n",
              "      <progress value='24' max='24' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
              "      [24/24 00:07]\n",
              "    </div>\n",
              "    "
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "   eval_loss  eval_accuracy   eval_f1  eval_precision  eval_recall  eval_runtime  \\\n",
              "0   0.491104          0.807  0.804326         0.80405        0.807        7.4435   \n",
              "\n",
              "   eval_samples_per_second  eval_steps_per_second  epoch  \n",
              "0                  403.035                  3.224   23.0  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-45f2b462-f58e-46ff-9b3e-badf3df05275\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>eval_loss</th>\n",
              "      <th>eval_accuracy</th>\n",
              "      <th>eval_f1</th>\n",
              "      <th>eval_precision</th>\n",
              "      <th>eval_recall</th>\n",
              "      <th>eval_runtime</th>\n",
              "      <th>eval_samples_per_second</th>\n",
              "      <th>eval_steps_per_second</th>\n",
              "      <th>epoch</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>0.491104</td>\n",
              "      <td>0.807</td>\n",
              "      <td>0.804326</td>\n",
              "      <td>0.80405</td>\n",
              "      <td>0.807</td>\n",
              "      <td>7.4435</td>\n",
              "      <td>403.035</td>\n",
              "      <td>3.224</td>\n",
              "      <td>23.0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-45f2b462-f58e-46ff-9b3e-badf3df05275')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-45f2b462-f58e-46ff-9b3e-badf3df05275 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-45f2b462-f58e-46ff-9b3e-badf3df05275');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "DYprp4X46mog"
      },
      "execution_count": 21,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Export Drive"
      ],
      "metadata": {
        "id": "eJrdPh8o-aRI"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "def COPY_File2Zip(Original_File_PATH, Copy_To_Directory, DELETE_ZIP=False, File_Name=None):  \n",
        "  if File_Name is None:  \n",
        "    File_Name = Original_File_PATH.split(\"/\")[-1] # Fold Name  \n",
        "  print(f\"Original_File_PATH: {Original_File_PATH}\\n ==>> Copy_To_Directory: {Copy_To_Directory}\")  \n",
        "  print(f\"*** CREATE {File_Name}.zip \", end=\"\")  \n",
        "  !zip -q -r {File_Name}.zip {Original_File_PATH}  \n",
        "  print(f\"*** COPY TO \", end=\"\")  \n",
        "  !cp -f /content/{File_Name}.zip {Copy_To_Directory}/{File_Name}.zip  \n",
        "  if DELETE_ZIP:  \n",
        "    print(f\"*** DELETE Original_ZIP \", end=\"\")  \n",
        "    !rm /content/{File_Name}.zip  \n",
        "  print(f\"*** DONE!\")"
      ],
      "metadata": {
        "id": "GyAvwNq4-YL8"
      },
      "execution_count": 22,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "if True:\n",
        "  Copy_To_Directory = \"/content/drive/MyDrive\"\n",
        "  dt_now = datetime.now(pytz.timezone('Asia/Tokyo'))\n",
        "  dttime = str(dt_now.year)+str(dt_now.month)+str(dt_now.day)+\"_\"+str(dt_now.hour)+\"_\"+str(dt_now.minute)+\"_\"+str(dt_now.second) \n",
        "  File_Name=f\"{dttime}_CDLE_NLP_TextClassification\"\n",
        "  COPY_File2Zip(MODELDIR, Copy_To_Directory, DELETE_ZIP=True, File_Name=File_Name)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Q9Qy8uMt-hBL",
        "outputId": "9afacb3f-e5da-4ccd-843d-bf3b9bf09a67"
      },
      "execution_count": 23,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Original_File_PATH: /content/CDLE_NLP/model\n",
            " ==>> Copy_To_Directory: /content/drive/MyDrive\n",
            "*** CREATE 202349_23_23_42_CDLE_NLP_TextClassification.zip *** COPY TO *** DELETE Original_ZIP *** DONE!\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "xXa6BrIMC2qk"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# inference"
      ],
      "metadata": {
        "id": "dytKJUEWai3S"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "## models load"
      ],
      "metadata": {
        "id": "6HNnlNu4akz8"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# model_path = '**SAVE PATH**'\n",
        "model_path = save_path_3"
      ],
      "metadata": {
        "id": "KyBWtfNq6mkY"
      },
      "execution_count": 24,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "new_tokenizer = LoadTokenizer(Config, pretrained=False, model_path=model_path)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "UNx90t8EgXpE",
        "outputId": "bac35c19-2e42-422a-c84a-06425b63985e"
      },
      "execution_count": 25,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "********************\n",
            "model path: /content/CDLE_NLP/model/202349-22-36-47-jarvisx17/japanese-sentiment-analysis-finetuned-emotion\n",
            "tokenizer語彙数: 32768, tokenizer最大長: 1000000000000000019884624838656\n",
            "tokenizer input_names: ['input_ids', 'token_type_ids', 'attention_mask']\n",
            "********************\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# new_model = CustomModelPooler(Config, pretrained=False, model_path=model_path)\n",
        "# new_model = CustomModelLastHiddenLayer(Config, pretrained=False, model_path=model_path)\n",
        "new_model = CustomModelMultiHiddenLayer(Config, use_multi_layer=-4, pretrained=False, model_path=model_path)\n",
        "# new_model = CustomModelLastHiddenLayerMeanPooling(Config, pretrained=False, model_path=model_path)\n",
        "# new_model = CustomModelLastHiddenLayerMaxPooling(Config, pretrained=False, model_path=model_path)\n",
        "# new_model = CustomModelLastHiddenLayerMeanMaxPooling(Config, pretrained=False, model_path=model_path)\n",
        "\n",
        "new_model.load_state_dict(torch.load(f\"{model_path}/pytorch_model.bin\"))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "V9YsZGMHqTE1",
        "outputId": "53a0e432-ca59-4b3a-e937-63a1a9f90d63"
      },
      "execution_count": 26,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "********************\n",
            "model path: /content/CDLE_NLP/model/202349-22-36-47-jarvisx17/japanese-sentiment-analysis-finetuned-emotion, device: cuda\n",
            "model hidden_size: 768, num_hidden_layers: 12\n",
            "=> num_classes: 3\n",
            "=> requires_grad is ALL False\n",
            "********************\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<All keys matched successfully>"
            ]
          },
          "metadata": {},
          "execution_count": 26
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "NvBDIth6wwJr"
      },
      "execution_count": 26,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## infer"
      ],
      "metadata": {
        "id": "w7Q-AVNZuV4t"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "remove_columns = ['text','source']\n",
        "test_dataset = create_dataset(test_df[['text','source']], new_tokenizer, cfg=Config, remove_columns=remove_columns)\n",
        "y_test = test_df['labels']"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 17,
          "referenced_widgets": [
            "2889e59cfc174efa87dd1f34a8df70ac",
            "ba355607e51f4a5bbef960020c6bc407",
            "5e47847e7cdb4039bc09e11c6ade7784",
            "0e94fc424c9a424db66bdc90859bbd17",
            "82fb0daf4380427e96010d831a702460",
            "bf76f2f518a04e5da9fc9942bb821859",
            "701b4f96599e429a982d3064babd4a7f",
            "05b5d1168a6540fd951c3fc904025aff",
            "20c37db485aa40a4be658581dbc970ff",
            "49d24d77165f491b9f74cec7843c2d49",
            "d254e8e4d7bb44699e3fdaf47a8eef4c"
          ]
        },
        "id": "KNMkfR5_jrKH",
        "outputId": "4abbb977-60ab-4b2b-9846-0868210f1eb4"
      },
      "execution_count": 27,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Running tokenizer on dataset:   0%|          | 0/3000 [00:00<?, ? examples/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "2889e59cfc174efa87dd1f34a8df70ac"
            }
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "inference_args = TrainingArguments(output_dir='/',         # 出力場所(inferenceは意味はない)\n",
        "                                   label_names=['labels'], # targetLabel( Default: labels )\n",
        "                                  #  fp16=True,\n",
        "                                   disable_tqdm=False,\n",
        "                                   push_to_hub=False,\n",
        "                                   report_to=['none'],\n",
        "                                   )\n",
        "inference_trainer = Trainer(model=new_model,\n",
        "                            args=inference_args,\n",
        "                            tokenizer=tokenizer,\n",
        "                            data_collator=DataCollatorWithPadding(\n",
        "                            tokenizer=tokenizer, padding=\"longest\"\n",
        "                            ),\n",
        "                            )\n",
        "\n",
        "test_predictions = []\n",
        "pred = inference_trainer.predict(test_dataset).predictions.astype(float)\n",
        "test_predictions.append(pred.squeeze())\n",
        "\n",
        "torch.cuda.empty_cache(); gc.collect();"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 17
        },
        "id": "tRQyRs1_oLrG",
        "outputId": "1c35dc0f-e7d7-47f4-95e1-8a127f2bb260"
      },
      "execution_count": 28,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": []
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "def metrics(y_true, y_prob):\n",
        "  y_pred = y_prob.argmax(-1)\n",
        "  acc = accuracy_score(y_true, y_pred)\n",
        "  f1 = f1_score(y_true, y_pred, average='weighted')\n",
        "  precision = precision_score(y_true, y_pred, average='weighted')\n",
        "  recall = recall_score(y_true, y_pred, average='weighted')\n",
        "  return {'accuracy': acc, 'f1': f1, 'precision': precision, 'recall': recall,}"
      ],
      "metadata": {
        "id": "TBHX5FFzsbpi"
      },
      "execution_count": 29,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "result = metrics(y_test, test_predictions[0])\n",
        "display(pd.DataFrame(result, index=[0]))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 81
        },
        "id": "pfZ-Kzf-o7ii",
        "outputId": "118bf361-d3dc-4f0e-f658-28f7ea8456e9"
      },
      "execution_count": 30,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "   accuracy        f1  precision    recall\n",
              "0  0.788667  0.788657   0.788654  0.788667"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-8d8bd909-fce9-4cba-a359-f596a1668275\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>accuracy</th>\n",
              "      <th>f1</th>\n",
              "      <th>precision</th>\n",
              "      <th>recall</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>0.788667</td>\n",
              "      <td>0.788657</td>\n",
              "      <td>0.788654</td>\n",
              "      <td>0.788667</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-8d8bd909-fce9-4cba-a359-f596a1668275')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-8d8bd909-fce9-4cba-a359-f596a1668275 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-8d8bd909-fce9-4cba-a359-f596a1668275');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "ayb36UIraihN"
      },
      "execution_count": 30,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# footer"
      ],
      "metadata": {
        "id": "u3JoRd1MapiJ"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# # ランタイムの終了\n",
        "# from google.colab import runtime\n",
        "# runtime.unassign()"
      ],
      "metadata": {
        "id": "03MPvI9_PDwQ"
      },
      "execution_count": 31,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "ikjLJWTpPDsx"
      },
      "execution_count": 31,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "0bJ_wwoFjXn7"
      },
      "execution_count": 31,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "V2REg9DjjXl5"
      },
      "execution_count": 31,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "ctSvn_7wjXeM"
      },
      "execution_count": 31,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "9QsKLcG-jXb4"
      },
      "execution_count": 31,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "Pg025hcS6Y4u"
      },
      "execution_count": 31,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "jmt6aKLy6Y1_"
      },
      "execution_count": 31,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "sTiGpjgs6Yza"
      },
      "execution_count": 31,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "axN_ly4yug2y"
      },
      "execution_count": 31,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "F8Qi_qauugsJ"
      },
      "execution_count": 31,
      "outputs": []
    }
  ]
}