ModernBERT_Fine_Tune_on_IMDB.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "gpuType": "T4"
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU",
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "09fcc248efd042c1a49b9486b269a654": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_f17b661f962d443c9afb1f68b4b722f6",
              "IPY_MODEL_dcdc3f4e745a4fb58278ca52e1b0b224",
              "IPY_MODEL_475f22baa21d4f5db9c20c102979e72d"
            ],
            "layout": "IPY_MODEL_9448b831eaf442db8c079df75ae52bb5"
          }
        },
        "f17b661f962d443c9afb1f68b4b722f6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_93fa77e6104f4bd190a197ceceed66f0",
            "placeholder": "​",
            "style": "IPY_MODEL_907930f2d78240cfab501799f6682f90",
            "value": "Map: 100%"
          }
        },
        "dcdc3f4e745a4fb58278ca52e1b0b224": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_78a743f8ea3342fe9894d640946cdf05",
            "max": 1500,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_afe8e3557d18476c9d9fdb564f43b583",
            "value": 1500
          }
        },
        "475f22baa21d4f5db9c20c102979e72d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_dd97e8943d284e3b871ff3bf0c5a14a9",
            "placeholder": "​",
            "style": "IPY_MODEL_2dd5089347d94b1cb7a5b3d954d3838c",
            "value": " 1500/1500 [00:04&lt;00:00, 334.83 examples/s]"
          }
        },
        "9448b831eaf442db8c079df75ae52bb5": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "93fa77e6104f4bd190a197ceceed66f0": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "907930f2d78240cfab501799f6682f90": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "78a743f8ea3342fe9894d640946cdf05": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "afe8e3557d18476c9d9fdb564f43b583": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "dd97e8943d284e3b871ff3bf0c5a14a9": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "2dd5089347d94b1cb7a5b3d954d3838c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "d96614fa0e4c422c97fbaf4e399988b3": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_aa6885bdaeb0479e90d5ff7683732a8a",
              "IPY_MODEL_cbcf9078bbc048ed9a626337ef3e9ea1",
              "IPY_MODEL_01fe0b2557d3420e8dff52aacd93498d"
            ],
            "layout": "IPY_MODEL_8fb2925d28c44e67a9f56699397930d5"
          }
        },
        "aa6885bdaeb0479e90d5ff7683732a8a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_116b336c5dac471f8960275c37f16c8b",
            "placeholder": "​",
            "style": "IPY_MODEL_b199703194b24425b161ff3752212244",
            "value": "Map: 100%"
          }
        },
        "cbcf9078bbc048ed9a626337ef3e9ea1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_1a9f9cd15fd24c16abf4112cb5d865e6",
            "max": 500,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_8a48eeb19b8740b880d5183f7d81e2bf",
            "value": 500
          }
        },
        "01fe0b2557d3420e8dff52aacd93498d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_c83599bb44884c5585d08aafee2958f0",
            "placeholder": "​",
            "style": "IPY_MODEL_49cb9acfc56f45c782e4732b172254a0",
            "value": " 500/500 [00:01&lt;00:00, 332.37 examples/s]"
          }
        },
        "8fb2925d28c44e67a9f56699397930d5": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "116b336c5dac471f8960275c37f16c8b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "b199703194b24425b161ff3752212244": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "1a9f9cd15fd24c16abf4112cb5d865e6": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "8a48eeb19b8740b880d5183f7d81e2bf": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "c83599bb44884c5585d08aafee2958f0": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "49cb9acfc56f45c782e4732b172254a0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "source": [
        "# ModernBERT: Fine-Tuning on the IMDB Dataset\n",
        "\n",
        "This notebook explores how to fine-tune [ModernBERT-base](https://huggingface.co/answerdotai/ModernBERT-base) using the [IMDB reviews dataset](https://huggingface.co/datasets/stanfordnlp/imdb).\n",
        "\n",
        "## Classification Task\n",
        "\n",
        "The IMDB reviews dataset contains `25,000` movie reviews binary coded for `negative` and `positive` sentiment. The task is to train a language model that can accurately predict whether a review is `positive` or `negative`.\n",
        "\n",
        "The features of the dataset are:\n",
        "\n",
        "| Feature | Description                  |\n",
        "|---------|------------------------------|\n",
        "| text    | str: the text of the review. |\n",
        "| label   | int: 1 = pos, 0 = neg        |\n"
      ],
      "metadata": {
        "id": "n7BaHB8VnWpZ"
      }
    },
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "0RcCoM7RmXOQ",
        "outputId": "1836d6cc-0111-40a7-bcb4-556f8ea162f5"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Requirement already satisfied: datasets in /usr/local/lib/python3.11/dist-packages (3.2.0)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.11/dist-packages (from datasets) (3.16.1)\n",
            "Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.11/dist-packages (from datasets) (1.26.4)\n",
            "Requirement already satisfied: pyarrow>=15.0.0 in /usr/local/lib/python3.11/dist-packages (from datasets) (17.0.0)\n",
            "Requirement already satisfied: dill<0.3.9,>=0.3.0 in /usr/local/lib/python3.11/dist-packages (from datasets) (0.3.8)\n",
            "Requirement already satisfied: pandas in /usr/local/lib/python3.11/dist-packages (from datasets) (2.2.2)\n",
            "Requirement already satisfied: requests>=2.32.2 in /usr/local/lib/python3.11/dist-packages (from datasets) (2.32.3)\n",
            "Requirement already satisfied: tqdm>=4.66.3 in /usr/local/lib/python3.11/dist-packages (from datasets) (4.67.1)\n",
            "Requirement already satisfied: xxhash in /usr/local/lib/python3.11/dist-packages (from datasets) (3.5.0)\n",
            "Requirement already satisfied: multiprocess<0.70.17 in /usr/local/lib/python3.11/dist-packages (from datasets) (0.70.16)\n",
            "Requirement already satisfied: fsspec<=2024.9.0,>=2023.1.0 in /usr/local/lib/python3.11/dist-packages (from fsspec[http]<=2024.9.0,>=2023.1.0->datasets) (2024.9.0)\n",
            "Requirement already satisfied: aiohttp in /usr/local/lib/python3.11/dist-packages (from datasets) (3.11.11)\n",
            "Requirement already satisfied: huggingface-hub>=0.23.0 in /usr/local/lib/python3.11/dist-packages (from datasets) (0.27.1)\n",
            "Requirement already satisfied: packaging in /usr/local/lib/python3.11/dist-packages (from datasets) (24.2)\n",
            "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.11/dist-packages (from datasets) (6.0.2)\n",
            "Requirement already satisfied: aiohappyeyeballs>=2.3.0 in /usr/local/lib/python3.11/dist-packages (from aiohttp->datasets) (2.4.4)\n",
            "Requirement already satisfied: aiosignal>=1.1.2 in /usr/local/lib/python3.11/dist-packages (from aiohttp->datasets) (1.3.2)\n",
            "Requirement already satisfied: attrs>=17.3.0 in /usr/local/lib/python3.11/dist-packages (from aiohttp->datasets) (24.3.0)\n",
            "Requirement already satisfied: frozenlist>=1.1.1 in /usr/local/lib/python3.11/dist-packages (from aiohttp->datasets) (1.5.0)\n",
            "Requirement already satisfied: multidict<7.0,>=4.5 in /usr/local/lib/python3.11/dist-packages (from aiohttp->datasets) (6.1.0)\n",
            "Requirement already satisfied: propcache>=0.2.0 in /usr/local/lib/python3.11/dist-packages (from aiohttp->datasets) (0.2.1)\n",
            "Requirement already satisfied: yarl<2.0,>=1.17.0 in /usr/local/lib/python3.11/dist-packages (from aiohttp->datasets) (1.18.3)\n",
            "Requirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.11/dist-packages (from huggingface-hub>=0.23.0->datasets) (4.12.2)\n",
            "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.11/dist-packages (from requests>=2.32.2->datasets) (3.4.1)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.11/dist-packages (from requests>=2.32.2->datasets) (3.10)\n",
            "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.11/dist-packages (from requests>=2.32.2->datasets) (2.3.0)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.11/dist-packages (from requests>=2.32.2->datasets) (2024.12.14)\n",
            "Requirement already satisfied: python-dateutil>=2.8.2 in /usr/local/lib/python3.11/dist-packages (from pandas->datasets) (2.8.2)\n",
            "Requirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.11/dist-packages (from pandas->datasets) (2024.2)\n",
            "Requirement already satisfied: tzdata>=2022.7 in /usr/local/lib/python3.11/dist-packages (from pandas->datasets) (2024.2)\n",
            "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.11/dist-packages (from python-dateutil>=2.8.2->pandas->datasets) (1.17.0)\n",
            "Requirement already satisfied: evaluate in /usr/local/lib/python3.11/dist-packages (0.4.3)\n",
            "Requirement already satisfied: datasets>=2.0.0 in /usr/local/lib/python3.11/dist-packages (from evaluate) (3.2.0)\n",
            "Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.11/dist-packages (from evaluate) (1.26.4)\n",
            "Requirement already satisfied: dill in /usr/local/lib/python3.11/dist-packages (from evaluate) (0.3.8)\n",
            "Requirement already satisfied: pandas in /usr/local/lib/python3.11/dist-packages (from evaluate) (2.2.2)\n",
            "Requirement already satisfied: requests>=2.19.0 in /usr/local/lib/python3.11/dist-packages (from evaluate) (2.32.3)\n",
            "Requirement already satisfied: tqdm>=4.62.1 in /usr/local/lib/python3.11/dist-packages (from evaluate) (4.67.1)\n",
            "Requirement already satisfied: xxhash in /usr/local/lib/python3.11/dist-packages (from evaluate) (3.5.0)\n",
            "Requirement already satisfied: multiprocess in /usr/local/lib/python3.11/dist-packages (from evaluate) (0.70.16)\n",
            "Requirement already satisfied: fsspec>=2021.05.0 in /usr/local/lib/python3.11/dist-packages (from fsspec[http]>=2021.05.0->evaluate) (2024.9.0)\n",
            "Requirement already satisfied: huggingface-hub>=0.7.0 in /usr/local/lib/python3.11/dist-packages (from evaluate) (0.27.1)\n",
            "Requirement already satisfied: packaging in /usr/local/lib/python3.11/dist-packages (from evaluate) (24.2)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.11/dist-packages (from datasets>=2.0.0->evaluate) (3.16.1)\n",
            "Requirement already satisfied: pyarrow>=15.0.0 in /usr/local/lib/python3.11/dist-packages (from datasets>=2.0.0->evaluate) (17.0.0)\n",
            "Requirement already satisfied: aiohttp in /usr/local/lib/python3.11/dist-packages (from datasets>=2.0.0->evaluate) (3.11.11)\n",
            "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.11/dist-packages (from datasets>=2.0.0->evaluate) (6.0.2)\n",
            "Requirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.11/dist-packages (from huggingface-hub>=0.7.0->evaluate) (4.12.2)\n",
            "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.11/dist-packages (from requests>=2.19.0->evaluate) (3.4.1)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.11/dist-packages (from requests>=2.19.0->evaluate) (3.10)\n",
            "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.11/dist-packages (from requests>=2.19.0->evaluate) (2.3.0)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.11/dist-packages (from requests>=2.19.0->evaluate) (2024.12.14)\n",
            "Requirement already satisfied: python-dateutil>=2.8.2 in /usr/local/lib/python3.11/dist-packages (from pandas->evaluate) (2.8.2)\n",
            "Requirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.11/dist-packages (from pandas->evaluate) (2024.2)\n",
            "Requirement already satisfied: tzdata>=2022.7 in /usr/local/lib/python3.11/dist-packages (from pandas->evaluate) (2024.2)\n",
            "Requirement already satisfied: aiohappyeyeballs>=2.3.0 in /usr/local/lib/python3.11/dist-packages (from aiohttp->datasets>=2.0.0->evaluate) (2.4.4)\n",
            "Requirement already satisfied: aiosignal>=1.1.2 in /usr/local/lib/python3.11/dist-packages (from aiohttp->datasets>=2.0.0->evaluate) (1.3.2)\n",
            "Requirement already satisfied: attrs>=17.3.0 in /usr/local/lib/python3.11/dist-packages (from aiohttp->datasets>=2.0.0->evaluate) (24.3.0)\n",
            "Requirement already satisfied: frozenlist>=1.1.1 in /usr/local/lib/python3.11/dist-packages (from aiohttp->datasets>=2.0.0->evaluate) (1.5.0)\n",
            "Requirement already satisfied: multidict<7.0,>=4.5 in /usr/local/lib/python3.11/dist-packages (from aiohttp->datasets>=2.0.0->evaluate) (6.1.0)\n",
            "Requirement already satisfied: propcache>=0.2.0 in /usr/local/lib/python3.11/dist-packages (from aiohttp->datasets>=2.0.0->evaluate) (0.2.1)\n",
            "Requirement already satisfied: yarl<2.0,>=1.17.0 in /usr/local/lib/python3.11/dist-packages (from aiohttp->datasets>=2.0.0->evaluate) (1.18.3)\n",
            "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.11/dist-packages (from python-dateutil>=2.8.2->pandas->evaluate) (1.17.0)\n"
          ]
        }
      ],
      "source": [
        "!pip install -U transformers>=4.48.0\n",
        "!pip install datasets\n",
        "!pip install evaluate"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# imports\n",
        "import numpy as np\n",
        "import pandas as pd\n",
        "\n",
        "import evaluate\n",
        "from transformers import (\n",
        "    AutoTokenizer,\n",
        "    AutoModelForSequenceClassification,\n",
        "    TrainingArguments,\n",
        "    Trainer\n",
        ")\n",
        "\n",
        "from datasets import load_dataset"
      ],
      "metadata": {
        "id": "I7TyihbUpBC8"
      },
      "execution_count": 2,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "**Grab dataset:**"
      ],
      "metadata": {
        "id": "wl6jgIEZqFWp"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# grab data:\n",
        "train_dataset = load_dataset(\"imdb\", split=\"train\")\n",
        "test_dataset = load_dataset(\"imdb\", split=\"test\")"
      ],
      "metadata": {
        "id": "YzXaIrQup_xA"
      },
      "execution_count": 3,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# sample:\n",
        "train_class_sample_size = 750\n",
        "test_class_sample_size = 250\n",
        "\n",
        "# training sample:\n",
        "pos_train_idx = np.where(np.array(train_dataset[\"label\"]) == 1)\n",
        "neg_train_idx = np.where(np.array(train_dataset[\"label\"]) == 0)\n",
        "pos_train_idx = np.random.choice(pos_train_idx[0], train_class_sample_size, replace=False)\n",
        "neg_train_idx = np.random.choice(neg_train_idx[0], train_class_sample_size, replace=False)\n",
        "train_dataset = train_dataset.select(np.sort(np.concatenate([pos_train_idx, neg_train_idx])))\n",
        "\n",
        "# testing sample:\n",
        "pos_test_idx = np.where(np.array(test_dataset[\"label\"]) == 1)\n",
        "neg_test_idx = np.where(np.array(test_dataset[\"label\"]) == 0)\n",
        "pos_test_idx = np.random.choice(pos_test_idx[0], test_class_sample_size, replace=False)\n",
        "neg_test_idx = np.random.choice(neg_test_idx[0], test_class_sample_size, replace=False)\n",
        "test_dataset = test_dataset.select(np.sort(np.concatenate([pos_test_idx, neg_test_idx])))"
      ],
      "metadata": {
        "id": "D9RLzYNHCn5N"
      },
      "execution_count": 4,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "train_dataset"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "XjWRLXOtDS_N",
        "outputId": "d600913d-90d2-4132-e8f1-7e3f6f9f6943"
      },
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "Dataset({\n",
              "    features: ['text', 'label'],\n",
              "    num_rows: 1500\n",
              "})"
            ]
          },
          "metadata": {},
          "execution_count": 5
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "test_dataset"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "v_e5J5lSqMBy",
        "outputId": "dce13ca3-9b53-492f-b30f-c583e5cd0b22"
      },
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "Dataset({\n",
              "    features: ['text', 'label'],\n",
              "    num_rows: 500\n",
              "})"
            ]
          },
          "metadata": {},
          "execution_count": 6
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "train_dataset[\"label\"][0], train_dataset[\"text\"][0]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "veGIKowIqPb7",
        "outputId": "dffd4c91-9e88-4258-a14c-934612459eb5"
      },
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(0,\n",
              " \"I have read all of the Love Come Softly books. Knowing full well that movies can not use all aspects of the book,but generally they at least have the main point of the book. I was highly disappointed in this movie. The only thing that they have in this movie that is in the book is that Missy's father comes to visit,(although in the book both parents come). That is all. The story line was so twisted and far fetch and yes, sad, from the book, that I just couldn't enjoy it. Even if I didn't read the book it was too sad. I do know that Pioneer life was rough,but the whole movie was a downer. The rating is for having the same family orientation of the film that makes them great.\")"
            ]
          },
          "metadata": {},
          "execution_count": 7
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "train_dataset[\"label\"][-1], train_dataset[\"text\"][-1]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "n0Jzx9TQD237",
        "outputId": "9a56105b-5873-405e-9bdc-77af1d74de49"
      },
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(1,\n",
              " \"Very smart, sometimes shocking, I just love it. It shoved one more side of David's brilliant talent. He impressed me greatly! David is the best. The movie captivates your attention for every second.\")"
            ]
          },
          "metadata": {},
          "execution_count": 8
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "test_dataset[\"label\"][0], test_dataset[\"text\"][0]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "CiCI3A_dqaUE",
        "outputId": "1434808b-a3c2-48fb-c296-eab0c024cd42"
      },
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(0,\n",
              " \"STAR RATING: ***** Saturday Night **** Friday Night *** Friday Morning ** Sunday Night * Monday Morning <br /><br />Former New Orleans homicide cop Jack Robideaux (Jean Claude Van Damme) is re-assigned to Columbus, a small but violent town in Mexico to help the police there with their efforts to stop a major heroin smuggling operation into their town. The culprits turn out to be ex-military, lead by former commander Benjamin Meyers (Stephen Lord, otherwise known as Jase from East Enders) who is using a special method he learned in Afghanistan to fight off his opponents. But Jack has a more personal reason for taking him down, that draws the two men into an explosive final showdown where only one will walk away alive.<br /><br />After Until Death, Van Damme appeared to be on a high, showing he could make the best straight to video films in the action market. While that was a far more drama oriented film, with The Shepherd he has returned to the high-kicking, no brainer action that first made him famous and has sadly produced his worst film since Derailed. It's nowhere near as bad as that film, but what I said still stands.<br /><br />A dull, predictable film, with very little in the way of any exciting action. What little there is mainly consists of some limp fight scenes, trying to look cool and trendy with some cheap slo-mo/sped up effects added to them that sadly instead make them look more desperate. Being a Mexican set film, director Isaac Florentine has tried to give the film a Robert Rodriguez/Desperado sort of feel, but this only adds to the desperation.<br /><br />VD gives a particularly uninspired performance and given he's never been a Robert De Niro sort of actor, that can't be good. As the villain, Lord shouldn't expect to leave the beeb anytime soon. He gets little dialogue at the beginning as he struggles to muster an American accent but gets mysteriously better towards the end. All the supporting cast are equally bland, and do nothing to raise the films spirits at all.<br /><br />This is one shepherd that's strayed right from the flock. *\")"
            ]
          },
          "metadata": {},
          "execution_count": 9
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "test_dataset[\"label\"][-1], test_dataset[\"text\"][-1]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "gFzyLVewEB5B",
        "outputId": "13442dbf-3811-4287-dc27-d490b877ce25"
      },
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(1,\n",
              " 'H.G. Cluozot had difficulties working in France after he had made \"Le Corbeau\" in 1943 which was produced by the German company and later judged by French as a piece of anti-French propaganda. Louis Jouvet, an admirer of Clouzot\\'s work, invited him to direct a thriller \"Quai des Orfevres\" where he played an ambiguous police inspector investigating a murder that happened in Paris Music Hall. Without each other knowledge, the seductive cabaret singer Jenny Lamoure (Suzy Delair) and her jealous piano-accompanist husband Maurice who is madly in love with her (Bertrand Blier, father of director Bertrand Blier) trying to cover up (without each other\\'s knowledge) what they believe to be their involvement in the murder? Enters tenacious policeman (Louis Jouvet) who is determined to discover the truth. Jouvet practically stole the movie with wonderfully cynic and sentimental in the same time performance. \"His character, his eagle-like profile and his unique way of speaking made him unforgettable.\" \"Quai des Orfevres\", witty and atmospheric observation of human weaknesses was a great comeback of H.G. Cluozot, the fine director, \"French Hitchcock\".')"
            ]
          },
          "metadata": {},
          "execution_count": 10
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Create classification label mappings:"
      ],
      "metadata": {
        "id": "9p7pPlWDrb5n"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "num_labels = 2\n",
        "id2label = {0: \"neg\", 1: \"pos\"}\n",
        "label2id = {v:k for k,v in id2label.items()}"
      ],
      "metadata": {
        "id": "NAtdrOCzrbaR"
      },
      "execution_count": 11,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "id2label"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "MGmRSqF3rmOP",
        "outputId": "1f759256-efbc-4a08-831b-2c80d0e7f2fd"
      },
      "execution_count": 12,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{0: 'neg', 1: 'pos'}"
            ]
          },
          "metadata": {},
          "execution_count": 12
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "label2id"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "dxVKgVfermz-",
        "outputId": "c1c264cb-3319-4195-9c90-ed722a1f7e68"
      },
      "execution_count": 13,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'neg': 0, 'pos': 1}"
            ]
          },
          "metadata": {},
          "execution_count": 13
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Model Features\n",
        "\n",
        "- Load the model, `ModernBERT-base`.\n",
        "- Load the model's tokenizer."
      ],
      "metadata": {
        "id": "QVKRhsG8qprV"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# model ID on Hugging Face:\n",
        "model_id = \"answerdotai/ModernBERT-base\"\n",
        "\n",
        "# model itself:\n",
        "model = AutoModelForSequenceClassification.from_pretrained(\n",
        "    model_id, num_labels=num_labels, id2label=id2label, label2id=label2id\n",
        ")\n",
        "\n",
        "# tokenizer:\n",
        "tokenizer = AutoTokenizer.from_pretrained(model_id)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "_7gEYEd7qg5x",
        "outputId": "883774b4-4fa4-4d88-e08c-58ecfa69a1b7"
      },
      "execution_count": 14,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Some weights of ModernBertForSequenceClassification were not initialized from the model checkpoint at answerdotai/ModernBERT-base and are newly initialized: ['classifier.bias', 'classifier.weight']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "model.to(\"cuda\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "MCOmKkAy8NwO",
        "outputId": "7c5e7210-9db3-420b-e8d7-811aa79c0fb1"
      },
      "execution_count": 15,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "ModernBertForSequenceClassification(\n",
              "  (model): ModernBertModel(\n",
              "    (embeddings): ModernBertEmbeddings(\n",
              "      (tok_embeddings): Embedding(50368, 768, padding_idx=50283)\n",
              "      (norm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
              "      (drop): Dropout(p=0.0, inplace=False)\n",
              "    )\n",
              "    (layers): ModuleList(\n",
              "      (0): ModernBertEncoderLayer(\n",
              "        (attn_norm): Identity()\n",
              "        (attn): ModernBertAttention(\n",
              "          (Wqkv): Linear(in_features=768, out_features=2304, bias=False)\n",
              "          (rotary_emb): ModernBertRotaryEmbedding()\n",
              "          (Wo): Linear(in_features=768, out_features=768, bias=False)\n",
              "          (out_drop): Identity()\n",
              "        )\n",
              "        (mlp_norm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
              "        (mlp): ModernBertMLP(\n",
              "          (Wi): Linear(in_features=768, out_features=2304, bias=False)\n",
              "          (act): GELUActivation()\n",
              "          (drop): Dropout(p=0.0, inplace=False)\n",
              "          (Wo): Linear(in_features=1152, out_features=768, bias=False)\n",
              "        )\n",
              "      )\n",
              "      (1-21): 21 x ModernBertEncoderLayer(\n",
              "        (attn_norm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
              "        (attn): ModernBertAttention(\n",
              "          (Wqkv): Linear(in_features=768, out_features=2304, bias=False)\n",
              "          (rotary_emb): ModernBertRotaryEmbedding()\n",
              "          (Wo): Linear(in_features=768, out_features=768, bias=False)\n",
              "          (out_drop): Identity()\n",
              "        )\n",
              "        (mlp_norm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
              "        (mlp): ModernBertMLP(\n",
              "          (Wi): Linear(in_features=768, out_features=2304, bias=False)\n",
              "          (act): GELUActivation()\n",
              "          (drop): Dropout(p=0.0, inplace=False)\n",
              "          (Wo): Linear(in_features=1152, out_features=768, bias=False)\n",
              "        )\n",
              "      )\n",
              "    )\n",
              "    (final_norm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
              "  )\n",
              "  (head): ModernBertPredictionHead(\n",
              "    (dense): Linear(in_features=768, out_features=768, bias=False)\n",
              "    (act): GELUActivation()\n",
              "    (norm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
              "  )\n",
              "  (drop): Dropout(p=0.0, inplace=False)\n",
              "  (classifier): Linear(in_features=768, out_features=2, bias=True)\n",
              ")"
            ]
          },
          "metadata": {},
          "execution_count": 15
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "def tokenize_function(examples):\n",
        "    return tokenizer(\n",
        "        examples[\"text\"],\n",
        "        truncation=True,\n",
        "        padding=True,\n",
        "        max_length=1024,\n",
        "        return_tensors=\"pt\"\n",
        "    )"
      ],
      "metadata": {
        "id": "9l9csD1orJ-9"
      },
      "execution_count": 16,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "training_tokenized = train_dataset.map(tokenize_function, batched=True, batch_size=1000)\n",
        "testing_tokenized = test_dataset.map(tokenize_function, batched=True, batch_size=1000)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 81,
          "referenced_widgets": [
            "09fcc248efd042c1a49b9486b269a654",
            "f17b661f962d443c9afb1f68b4b722f6",
            "dcdc3f4e745a4fb58278ca52e1b0b224",
            "475f22baa21d4f5db9c20c102979e72d",
            "9448b831eaf442db8c079df75ae52bb5",
            "93fa77e6104f4bd190a197ceceed66f0",
            "907930f2d78240cfab501799f6682f90",
            "78a743f8ea3342fe9894d640946cdf05",
            "afe8e3557d18476c9d9fdb564f43b583",
            "dd97e8943d284e3b871ff3bf0c5a14a9",
            "2dd5089347d94b1cb7a5b3d954d3838c",
            "d96614fa0e4c422c97fbaf4e399988b3",
            "aa6885bdaeb0479e90d5ff7683732a8a",
            "cbcf9078bbc048ed9a626337ef3e9ea1",
            "01fe0b2557d3420e8dff52aacd93498d",
            "8fb2925d28c44e67a9f56699397930d5",
            "116b336c5dac471f8960275c37f16c8b",
            "b199703194b24425b161ff3752212244",
            "1a9f9cd15fd24c16abf4112cb5d865e6",
            "8a48eeb19b8740b880d5183f7d81e2bf",
            "c83599bb44884c5585d08aafee2958f0",
            "49cb9acfc56f45c782e4732b172254a0"
          ]
        },
        "id": "sb-tlYmisw0c",
        "outputId": "86f2a9d4-db4c-4481-a6ca-a355579ada3b"
      },
      "execution_count": 17,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Map:   0%|          | 0/1500 [00:00<?, ? examples/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "09fcc248efd042c1a49b9486b269a654"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Map:   0%|          | 0/500 [00:00<?, ? examples/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "d96614fa0e4c422c97fbaf4e399988b3"
            }
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Train"
      ],
      "metadata": {
        "id": "OZjxBJg8vldP"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# metrics\n",
        "accuracy = evaluate.load(\"accuracy\")\n",
        "precision = evaluate.load(\"precision\")\n",
        "recall = evaluate.load(\"recall\")\n",
        "f1_score = evaluate.load(\"f1\")"
      ],
      "metadata": {
        "id": "ghDT_gv1s2p5"
      },
      "execution_count": 18,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def compute_metrics(eval_pred):\n",
        "    # logits, labels, and predictions:\n",
        "    logits, labels = eval_pred\n",
        "    predictions = np.argmax(logits, axis=1)\n",
        "\n",
        "    acc = accuracy.compute(predictions=predictions, references=labels)[\"accuracy\"]\n",
        "    pre = precision.compute(predictions=predictions, references=labels)[\"precision\"]\n",
        "    rec = recall.compute(predictions=predictions, references=labels)[\"recall\"]\n",
        "    f1 = f1_score.compute(predictions=predictions, references=labels, average=\"binary\")[\"f1\"]\n",
        "\n",
        "    return {\"accuracy\": acc, \"precision\": pre, \"recall\": rec, \"f1\": f1}"
      ],
      "metadata": {
        "id": "pPfCjfz8woVa"
      },
      "execution_count": 19,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# training arguments\n",
        "training_args = TrainingArguments(\n",
        "    output_dir=\"ModernBERT-imdb-classifier\",\n",
        "    per_device_train_batch_size=4,\n",
        "    per_device_eval_batch_size=2,\n",
        "    learning_rate=8e-5,\n",
        "    num_train_epochs=3,\n",
        "    logging_strategy=\"steps\",\n",
        "    logging_steps=100,\n",
        "    eval_strategy=\"epoch\",\n",
        "    save_strategy=\"epoch\",\n",
        "    save_total_limit=2,\n",
        "    load_best_model_at_end=True,\n",
        "    report_to=\"none\"\n",
        ")"
      ],
      "metadata": {
        "id": "D0x7CSg0vwuB"
      },
      "execution_count": 20,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# trainer:\n",
        "trainer = Trainer(\n",
        "    model=model,\n",
        "    args=training_args,\n",
        "    train_dataset=training_tokenized,\n",
        "    eval_dataset=testing_tokenized,\n",
        "    compute_metrics=compute_metrics\n",
        ")"
      ],
      "metadata": {
        "id": "AQZvC7nnw8Kq"
      },
      "execution_count": 21,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "trainer.train()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 222
        },
        "id": "WUCW_75PxO_o",
        "outputId": "2ad72eba-ab3d-4b60-d64c-ae3f57139851"
      },
      "execution_count": 22,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              "\n",
              "    <div>\n",
              "      \n",
              "      <progress value='1125' max='1125' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
              "      [1125/1125 29:23, Epoch 3/3]\n",
              "    </div>\n",
              "    <table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              " <tr style=\"text-align: left;\">\n",
              "      <th>Epoch</th>\n",
              "      <th>Training Loss</th>\n",
              "      <th>Validation Loss</th>\n",
              "      <th>Accuracy</th>\n",
              "      <th>Precision</th>\n",
              "      <th>Recall</th>\n",
              "      <th>F1</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <td>1</td>\n",
              "      <td>0.391000</td>\n",
              "      <td>0.460599</td>\n",
              "      <td>0.896000</td>\n",
              "      <td>0.843750</td>\n",
              "      <td>0.972000</td>\n",
              "      <td>0.903346</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2</td>\n",
              "      <td>0.155700</td>\n",
              "      <td>0.294641</td>\n",
              "      <td>0.948000</td>\n",
              "      <td>0.940945</td>\n",
              "      <td>0.956000</td>\n",
              "      <td>0.948413</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3</td>\n",
              "      <td>0.066700</td>\n",
              "      <td>0.281353</td>\n",
              "      <td>0.954000</td>\n",
              "      <td>0.959514</td>\n",
              "      <td>0.948000</td>\n",
              "      <td>0.953722</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table><p>"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "TrainOutput(global_step=1125, training_loss=0.24709596106720466, metrics={'train_runtime': 1774.1529, 'train_samples_per_second': 2.536, 'train_steps_per_second': 0.634, 'total_flos': 3066820614144000.0, 'train_loss': 0.24709596106720466, 'epoch': 3.0})"
            ]
          },
          "metadata": {},
          "execution_count": 22
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "trainer.save_model(\"ModernBERT-imdb-classifier\")"
      ],
      "metadata": {
        "id": "MMYLxHfnQB8Y"
      },
      "execution_count": 23,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "metric_df = pd.DataFrame(trainer.state.log_history)\n",
        "metric_df"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 540
        },
        "id": "5Iej3FbDQQxa",
        "outputId": "b2bfc919-9d82-4596-eb34-4018097914f1"
      },
      "execution_count": 24,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "      loss  grad_norm  learning_rate     epoch  step  eval_loss  \\\n",
              "0   0.8877  71.058342       0.000073  0.266667   100        NaN   \n",
              "1   0.5248  25.423561       0.000066  0.533333   200        NaN   \n",
              "2   0.3910  51.821434       0.000059  0.800000   300        NaN   \n",
              "3      NaN        NaN            NaN  1.000000   375   0.460599   \n",
              "4   0.3223   8.487826       0.000052  1.066667   400        NaN   \n",
              "5   0.1133   0.008962       0.000044  1.333333   500        NaN   \n",
              "6   0.2199   0.014823       0.000037  1.600000   600        NaN   \n",
              "7   0.1557   0.006990       0.000030  1.866667   700        NaN   \n",
              "8      NaN        NaN            NaN  2.000000   750   0.294641   \n",
              "9   0.0622   0.004130       0.000023  2.133333   800        NaN   \n",
              "10  0.0248   0.010613       0.000016  2.400000   900        NaN   \n",
              "11  0.0114   0.001263       0.000009  2.666667  1000        NaN   \n",
              "12  0.0667   0.002702       0.000002  2.933333  1100        NaN   \n",
              "13     NaN        NaN            NaN  3.000000  1125   0.281353   \n",
              "14     NaN        NaN            NaN  3.000000  1125        NaN   \n",
              "\n",
              "    eval_accuracy  eval_precision  eval_recall   eval_f1  eval_runtime  \\\n",
              "0             NaN             NaN          NaN       NaN           NaN   \n",
              "1             NaN             NaN          NaN       NaN           NaN   \n",
              "2             NaN             NaN          NaN       NaN           NaN   \n",
              "3           0.896        0.843750        0.972  0.903346       53.0340   \n",
              "4             NaN             NaN          NaN       NaN           NaN   \n",
              "5             NaN             NaN          NaN       NaN           NaN   \n",
              "6             NaN             NaN          NaN       NaN           NaN   \n",
              "7             NaN             NaN          NaN       NaN           NaN   \n",
              "8           0.948        0.940945        0.956  0.948413       52.2923   \n",
              "9             NaN             NaN          NaN       NaN           NaN   \n",
              "10            NaN             NaN          NaN       NaN           NaN   \n",
              "11            NaN             NaN          NaN       NaN           NaN   \n",
              "12            NaN             NaN          NaN       NaN           NaN   \n",
              "13          0.954        0.959514        0.948  0.953722       51.4055   \n",
              "14            NaN             NaN          NaN       NaN           NaN   \n",
              "\n",
              "    eval_samples_per_second  eval_steps_per_second  train_runtime  \\\n",
              "0                       NaN                    NaN            NaN   \n",
              "1                       NaN                    NaN            NaN   \n",
              "2                       NaN                    NaN            NaN   \n",
              "3                     9.428                  4.714            NaN   \n",
              "4                       NaN                    NaN            NaN   \n",
              "5                       NaN                    NaN            NaN   \n",
              "6                       NaN                    NaN            NaN   \n",
              "7                       NaN                    NaN            NaN   \n",
              "8                     9.562                  4.781            NaN   \n",
              "9                       NaN                    NaN            NaN   \n",
              "10                      NaN                    NaN            NaN   \n",
              "11                      NaN                    NaN            NaN   \n",
              "12                      NaN                    NaN            NaN   \n",
              "13                    9.727                  4.863            NaN   \n",
              "14                      NaN                    NaN      1774.1529   \n",
              "\n",
              "    train_samples_per_second  train_steps_per_second    total_flos  train_loss  \n",
              "0                        NaN                     NaN           NaN         NaN  \n",
              "1                        NaN                     NaN           NaN         NaN  \n",
              "2                        NaN                     NaN           NaN         NaN  \n",
              "3                        NaN                     NaN           NaN         NaN  \n",
              "4                        NaN                     NaN           NaN         NaN  \n",
              "5                        NaN                     NaN           NaN         NaN  \n",
              "6                        NaN                     NaN           NaN         NaN  \n",
              "7                        NaN                     NaN           NaN         NaN  \n",
              "8                        NaN                     NaN           NaN         NaN  \n",
              "9                        NaN                     NaN           NaN         NaN  \n",
              "10                       NaN                     NaN           NaN         NaN  \n",
              "11                       NaN                     NaN           NaN         NaN  \n",
              "12                       NaN                     NaN           NaN         NaN  \n",
              "13                       NaN                     NaN           NaN         NaN  \n",
              "14                     2.536                   0.634  3.066821e+15    0.247096  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-61348b82-6a42-403a-b2b2-c891002b6383\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>loss</th>\n",
              "      <th>grad_norm</th>\n",
              "      <th>learning_rate</th>\n",
              "      <th>epoch</th>\n",
              "      <th>step</th>\n",
              "      <th>eval_loss</th>\n",
              "      <th>eval_accuracy</th>\n",
              "      <th>eval_precision</th>\n",
              "      <th>eval_recall</th>\n",
              "      <th>eval_f1</th>\n",
              "      <th>eval_runtime</th>\n",
              "      <th>eval_samples_per_second</th>\n",
              "      <th>eval_steps_per_second</th>\n",
              "      <th>train_runtime</th>\n",
              "      <th>train_samples_per_second</th>\n",
              "      <th>train_steps_per_second</th>\n",
              "      <th>total_flos</th>\n",
              "      <th>train_loss</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>0.8877</td>\n",
              "      <td>71.058342</td>\n",
              "      <td>0.000073</td>\n",
              "      <td>0.266667</td>\n",
              "      <td>100</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>0.5248</td>\n",
              "      <td>25.423561</td>\n",
              "      <td>0.000066</td>\n",
              "      <td>0.533333</td>\n",
              "      <td>200</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>0.3910</td>\n",
              "      <td>51.821434</td>\n",
              "      <td>0.000059</td>\n",
              "      <td>0.800000</td>\n",
              "      <td>300</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>375</td>\n",
              "      <td>0.460599</td>\n",
              "      <td>0.896</td>\n",
              "      <td>0.843750</td>\n",
              "      <td>0.972</td>\n",
              "      <td>0.903346</td>\n",
              "      <td>53.0340</td>\n",
              "      <td>9.428</td>\n",
              "      <td>4.714</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>0.3223</td>\n",
              "      <td>8.487826</td>\n",
              "      <td>0.000052</td>\n",
              "      <td>1.066667</td>\n",
              "      <td>400</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>0.1133</td>\n",
              "      <td>0.008962</td>\n",
              "      <td>0.000044</td>\n",
              "      <td>1.333333</td>\n",
              "      <td>500</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6</th>\n",
              "      <td>0.2199</td>\n",
              "      <td>0.014823</td>\n",
              "      <td>0.000037</td>\n",
              "      <td>1.600000</td>\n",
              "      <td>600</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7</th>\n",
              "      <td>0.1557</td>\n",
              "      <td>0.006990</td>\n",
              "      <td>0.000030</td>\n",
              "      <td>1.866667</td>\n",
              "      <td>700</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8</th>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>2.000000</td>\n",
              "      <td>750</td>\n",
              "      <td>0.294641</td>\n",
              "      <td>0.948</td>\n",
              "      <td>0.940945</td>\n",
              "      <td>0.956</td>\n",
              "      <td>0.948413</td>\n",
              "      <td>52.2923</td>\n",
              "      <td>9.562</td>\n",
              "      <td>4.781</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9</th>\n",
              "      <td>0.0622</td>\n",
              "      <td>0.004130</td>\n",
              "      <td>0.000023</td>\n",
              "      <td>2.133333</td>\n",
              "      <td>800</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>10</th>\n",
              "      <td>0.0248</td>\n",
              "      <td>0.010613</td>\n",
              "      <td>0.000016</td>\n",
              "      <td>2.400000</td>\n",
              "      <td>900</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>11</th>\n",
              "      <td>0.0114</td>\n",
              "      <td>0.001263</td>\n",
              "      <td>0.000009</td>\n",
              "      <td>2.666667</td>\n",
              "      <td>1000</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>12</th>\n",
              "      <td>0.0667</td>\n",
              "      <td>0.002702</td>\n",
              "      <td>0.000002</td>\n",
              "      <td>2.933333</td>\n",
              "      <td>1100</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>13</th>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>3.000000</td>\n",
              "      <td>1125</td>\n",
              "      <td>0.281353</td>\n",
              "      <td>0.954</td>\n",
              "      <td>0.959514</td>\n",
              "      <td>0.948</td>\n",
              "      <td>0.953722</td>\n",
              "      <td>51.4055</td>\n",
              "      <td>9.727</td>\n",
              "      <td>4.863</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>14</th>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>3.000000</td>\n",
              "      <td>1125</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>1774.1529</td>\n",
              "      <td>2.536</td>\n",
              "      <td>0.634</td>\n",
              "      <td>3.066821e+15</td>\n",
              "      <td>0.247096</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-61348b82-6a42-403a-b2b2-c891002b6383')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-61348b82-6a42-403a-b2b2-c891002b6383 button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-61348b82-6a42-403a-b2b2-c891002b6383');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-4427cb6f-2c1b-4c3a-8475-38943e01215b\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-4427cb6f-2c1b-4c3a-8475-38943e01215b')\"\n",
              "            title=\"Suggest charts\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-4427cb6f-2c1b-4c3a-8475-38943e01215b button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "\n",
              "  <div id=\"id_0e6cc67a-9a3f-4a6e-968c-cd06e09db771\">\n",
              "    <style>\n",
              "      .colab-df-generate {\n",
              "        background-color: #E8F0FE;\n",
              "        border: none;\n",
              "        border-radius: 50%;\n",
              "        cursor: pointer;\n",
              "        display: none;\n",
              "        fill: #1967D2;\n",
              "        height: 32px;\n",
              "        padding: 0 0 0 0;\n",
              "        width: 32px;\n",
              "      }\n",
              "\n",
              "      .colab-df-generate:hover {\n",
              "        background-color: #E2EBFA;\n",
              "        box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "        fill: #174EA6;\n",
              "      }\n",
              "\n",
              "      [theme=dark] .colab-df-generate {\n",
              "        background-color: #3B4455;\n",
              "        fill: #D2E3FC;\n",
              "      }\n",
              "\n",
              "      [theme=dark] .colab-df-generate:hover {\n",
              "        background-color: #434B5C;\n",
              "        box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "        filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "        fill: #FFFFFF;\n",
              "      }\n",
              "    </style>\n",
              "    <button class=\"colab-df-generate\" onclick=\"generateWithVariable('metric_df')\"\n",
              "            title=\"Generate code using this dataframe.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M7,19H8.4L18.45,9,17,7.55,7,17.6ZM5,21V16.75L18.45,3.32a2,2,0,0,1,2.83,0l1.4,1.43a1.91,1.91,0,0,1,.58,1.4,1.91,1.91,0,0,1-.58,1.4L9.25,21ZM18.45,9,17,7.55Zm-12,3A5.31,5.31,0,0,0,4.9,8.1,5.31,5.31,0,0,0,1,6.5,5.31,5.31,0,0,0,4.9,4.9,5.31,5.31,0,0,0,6.5,1,5.31,5.31,0,0,0,8.1,4.9,5.31,5.31,0,0,0,12,6.5,5.46,5.46,0,0,0,6.5,12Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "    <script>\n",
              "      (() => {\n",
              "      const buttonEl =\n",
              "        document.querySelector('#id_0e6cc67a-9a3f-4a6e-968c-cd06e09db771 button.colab-df-generate');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      buttonEl.onclick = () => {\n",
              "        google.colab.notebook.generateWithVariable('metric_df');\n",
              "      }\n",
              "      })();\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "    </div>\n",
              "  </div>\n"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "dataframe",
              "variable_name": "metric_df",
              "summary": "{\n  \"name\": \"metric_df\",\n  \"rows\": 15,\n  \"fields\": [\n    {\n      \"column\": \"loss\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 0.26687113165175985,\n        \"min\": 0.0114,\n        \"max\": 0.8877,\n        \"num_unique_values\": 11,\n        \"samples\": [\n          0.2199,\n          0.8877,\n          0.0114\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"grad_norm\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 24.933668943896148,\n        \"min\": 0.001262593432329595,\n        \"max\": 71.05834197998047,\n        \"num_unique_values\": 11,\n        \"samples\": [\n          0.01482314895838499,\n          71.05834197998047,\n          0.001262593432329595\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"learning_rate\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 2.3584887398082842e-05,\n        \"min\": 1.777777777777778e-06,\n        \"max\": 7.28888888888889e-05,\n        \"num_unique_values\": 11,\n        \"samples\": [\n          3.733333333333334e-05,\n          7.28888888888889e-05,\n          8.888888888888888e-06\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"epoch\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 0.9185219320639761,\n        \"min\": 0.26666666666666666,\n        \"max\": 3.0,\n        \"num_unique_values\": 14,\n        \"samples\": [\n          2.1333333333333333,\n          2.6666666666666665,\n          0.26666666666666666\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"step\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 344,\n        \"min\": 100,\n        \"max\": 1125,\n        \"num_unique_values\": 14,\n        \"samples\": [\n          800,\n          1000,\n          100\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"eval_loss\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 0.09987350420847892,\n        \"min\": 0.28135251998901367,\n        \"max\": 0.460599422454834,\n        \"num_unique_values\": 3,\n        \"samples\": [\n          0.460599422454834,\n          0.29464077949523926,\n          0.28135251998901367\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"eval_accuracy\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 0.031895663237081796,\n        \"min\": 0.896,\n        \"max\": 0.954,\n        \"num_unique_values\": 3,\n        \"samples\": [\n          0.896,\n          0.948,\n          0.954\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"eval_precision\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 0.062173154165272145,\n        \"min\": 0.84375,\n        \"max\": 0.9595141700404858,\n        \"num_unique_values\": 3,\n        \"samples\": [\n          0.84375,\n          0.9409448818897638,\n          0.9595141700404858\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"eval_recall\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 0.012220201853215583,\n        \"min\": 0.948,\n        \"max\": 0.972,\n        \"num_unique_values\": 3,\n        \"samples\": [\n          0.972,\n          0.956,\n          0.948\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"eval_f1\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 0.02767979731133131,\n        \"min\": 0.9033457249070632,\n        \"max\": 0.9537223340040242,\n        \"num_unique_values\": 3,\n        \"samples\": [\n          0.9033457249070632,\n          0.9484126984126984,\n          0.9537223340040242\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"eval_runtime\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 0.8153266605068973,\n        \"min\": 51.4055,\n        \"max\": 53.034,\n        \"num_unique_values\": 3,\n        \"samples\": [\n          53.034,\n          52.2923,\n          51.4055\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"eval_samples_per_second\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 0.14976759774174542,\n        \"min\": 9.428,\n        \"max\": 9.727,\n        \"num_unique_values\": 3,\n        \"samples\": [\n          9.428,\n          9.562,\n          9.727\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"eval_steps_per_second\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 0.07462573282722257,\n        \"min\": 4.714,\n        \"max\": 4.863,\n        \"num_unique_values\": 3,\n        \"samples\": [\n          4.714,\n          4.781,\n          4.863\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"train_runtime\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": null,\n        \"min\": 1774.1529,\n        \"max\": 1774.1529,\n        \"num_unique_values\": 1,\n        \"samples\": [\n          1774.1529\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"train_samples_per_second\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": null,\n        \"min\": 2.536,\n        \"max\": 2.536,\n        \"num_unique_values\": 1,\n        \"samples\": [\n          2.536\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"train_steps_per_second\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": null,\n        \"min\": 0.634,\n        \"max\": 0.634,\n        \"num_unique_values\": 1,\n        \"samples\": [\n          0.634\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"total_flos\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": null,\n        \"min\": 3066820614144000.0,\n        \"max\": 3066820614144000.0,\n        \"num_unique_values\": 1,\n        \"samples\": [\n          3066820614144000.0\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"train_loss\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": null,\n        \"min\": 0.24709596106720466,\n        \"max\": 0.24709596106720466,\n        \"num_unique_values\": 1,\n        \"samples\": [\n          0.24709596106720466\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    }\n  ]\n}"
            }
          },
          "metadata": {},
          "execution_count": 24
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "trainer.evaluate()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 193
        },
        "id": "KxYQrQx8QZax",
        "outputId": "91b5a657-01b4-4247-ef70-779c74f1c8e9"
      },
      "execution_count": 25,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              "\n",
              "    <div>\n",
              "      \n",
              "      <progress value='250' max='250' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
              "      [250/250 00:51]\n",
              "    </div>\n",
              "    "
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'eval_loss': 0.28135251998901367,\n",
              " 'eval_accuracy': 0.954,\n",
              " 'eval_precision': 0.9595141700404858,\n",
              " 'eval_recall': 0.948,\n",
              " 'eval_f1': 0.9537223340040242,\n",
              " 'eval_runtime': 52.0983,\n",
              " 'eval_samples_per_second': 9.597,\n",
              " 'eval_steps_per_second': 4.799,\n",
              " 'epoch': 3.0}"
            ]
          },
          "metadata": {},
          "execution_count": 25
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "ECH6dd0KQqtq"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}