tinyllama_code.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/mohanad-hafez/tinyllama_code_finetuning/blob/main/tinyllama_code.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {
        "id": "ngSUcq9uAG9O"
      },
      "outputs": [],
      "source": [
        "import torch\n",
        "# Check so there is a gpu available, a T4(free tier) is enough to run this notebook\n",
        "assert (torch.cuda.is_available()==True)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 2,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "AG3wcM7uAG9P",
        "outputId": "6891d57f-e3ca-49e9-aeb8-2c39e3a48eca"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Requirement already satisfied: axolotl[deepspeed] in /usr/local/lib/python3.10/dist-packages (0.6.0)\n",
            "Requirement already satisfied: bitsandbytes==0.45.0 in /usr/local/lib/python3.10/dist-packages (from axolotl[deepspeed]) (0.45.0)\n",
            "Requirement already satisfied: triton>=2.3.0 in /usr/local/lib/python3.10/dist-packages (from axolotl[deepspeed]) (3.1.0)\n",
            "Requirement already satisfied: liger-kernel==0.4.2 in /usr/local/lib/python3.10/dist-packages (from axolotl[deepspeed]) (0.4.2)\n",
            "Requirement already satisfied: packaging==23.2 in /usr/local/lib/python3.10/dist-packages (from axolotl[deepspeed]) (23.2)\n",
            "Requirement already satisfied: peft==0.14.0 in /usr/local/lib/python3.10/dist-packages (from axolotl[deepspeed]) (0.14.0)\n",
            "Requirement already satisfied: transformers>=4.46.3 in /usr/local/lib/python3.10/dist-packages (from axolotl[deepspeed]) (4.47.1)\n",
            "Requirement already satisfied: tokenizers>=0.20.1 in /usr/local/lib/python3.10/dist-packages (from axolotl[deepspeed]) (0.21.0)\n",
            "Requirement already satisfied: accelerate==1.2.0 in /usr/local/lib/python3.10/dist-packages (from axolotl[deepspeed]) (1.2.0)\n",
            "Requirement already satisfied: datasets==3.1.0 in /usr/local/lib/python3.10/dist-packages (from axolotl[deepspeed]) (3.1.0)\n",
            "Requirement already satisfied: pydantic==2.6.3 in /usr/local/lib/python3.10/dist-packages (from axolotl[deepspeed]) (2.6.3)\n",
            "Requirement already satisfied: addict in /usr/local/lib/python3.10/dist-packages (from axolotl[deepspeed]) (2.4.0)\n",
            "Requirement already satisfied: fire in /usr/local/lib/python3.10/dist-packages (from axolotl[deepspeed]) (0.7.0)\n",
            "Requirement already satisfied: PyYAML>=6.0 in /usr/local/lib/python3.10/dist-packages (from axolotl[deepspeed]) (6.0.2)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.10/dist-packages (from axolotl[deepspeed]) (2.32.3)\n",
            "Requirement already satisfied: sentencepiece in /usr/local/lib/python3.10/dist-packages (from axolotl[deepspeed]) (0.2.0)\n",
            "Requirement already satisfied: wandb in /usr/local/lib/python3.10/dist-packages (from axolotl[deepspeed]) (0.19.1)\n",
            "Requirement already satisfied: einops in /usr/local/lib/python3.10/dist-packages (from axolotl[deepspeed]) (0.8.0)\n",
            "Requirement already satisfied: optimum==1.16.2 in /usr/local/lib/python3.10/dist-packages (from axolotl[deepspeed]) (1.16.2)\n",
            "Requirement already satisfied: hf-transfer in /usr/local/lib/python3.10/dist-packages (from axolotl[deepspeed]) (0.1.8)\n",
            "Requirement already satisfied: colorama in /usr/local/lib/python3.10/dist-packages (from axolotl[deepspeed]) (0.4.6)\n",
            "Requirement already satisfied: numba in /usr/local/lib/python3.10/dist-packages (from axolotl[deepspeed]) (0.60.0)\n",
            "Requirement already satisfied: numpy<=2.0.1,>=1.24.4 in /usr/local/lib/python3.10/dist-packages (from axolotl[deepspeed]) (1.26.4)\n",
            "Requirement already satisfied: evaluate==0.4.1 in /usr/local/lib/python3.10/dist-packages (from axolotl[deepspeed]) (0.4.1)\n",
            "Requirement already satisfied: scipy in /usr/local/lib/python3.10/dist-packages (from axolotl[deepspeed]) (1.13.1)\n",
            "Requirement already satisfied: scikit-learn==1.4.2 in /usr/local/lib/python3.10/dist-packages (from axolotl[deepspeed]) (1.4.2)\n",
            "Requirement already satisfied: nvidia-ml-py==12.560.30 in /usr/local/lib/python3.10/dist-packages (from axolotl[deepspeed]) (12.560.30)\n",
            "Requirement already satisfied: art in /usr/local/lib/python3.10/dist-packages (from axolotl[deepspeed]) (6.4)\n",
            "Requirement already satisfied: gradio==3.50.2 in /usr/local/lib/python3.10/dist-packages (from axolotl[deepspeed]) (3.50.2)\n",
            "Requirement already satisfied: tensorboard in /usr/local/lib/python3.10/dist-packages (from axolotl[deepspeed]) (2.17.1)\n",
            "Requirement already satisfied: python-dotenv==1.0.1 in /usr/local/lib/python3.10/dist-packages (from axolotl[deepspeed]) (1.0.1)\n",
            "Requirement already satisfied: s3fs>=2024.5.0 in /usr/local/lib/python3.10/dist-packages (from axolotl[deepspeed]) (2024.9.0)\n",
            "Requirement already satisfied: gcsfs>=2024.5.0 in /usr/local/lib/python3.10/dist-packages (from axolotl[deepspeed]) (2024.9.0.post1)\n",
            "Requirement already satisfied: trl==0.12.1 in /usr/local/lib/python3.10/dist-packages (from axolotl[deepspeed]) (0.12.1)\n",
            "Requirement already satisfied: zstandard==0.22.0 in /usr/local/lib/python3.10/dist-packages (from axolotl[deepspeed]) (0.22.0)\n",
            "Requirement already satisfied: fastcore in /usr/local/lib/python3.10/dist-packages (from axolotl[deepspeed]) (1.7.27)\n",
            "Requirement already satisfied: lm-eval==0.4.4 in /usr/local/lib/python3.10/dist-packages (from axolotl[deepspeed]) (0.4.4)\n",
            "Requirement already satisfied: langdetect==1.0.9 in /usr/local/lib/python3.10/dist-packages (from axolotl[deepspeed]) (1.0.9)\n",
            "Requirement already satisfied: immutabledict==4.2.0 in /usr/local/lib/python3.10/dist-packages (from axolotl[deepspeed]) (4.2.0)\n",
            "Requirement already satisfied: antlr4-python3-runtime==4.13.2 in /usr/local/lib/python3.10/dist-packages (from axolotl[deepspeed]) (4.13.2)\n",
            "Requirement already satisfied: torchao==0.5.0 in /usr/local/lib/python3.10/dist-packages (from axolotl[deepspeed]) (0.5.0)\n",
            "Requirement already satisfied: schedulefree==1.3.0 in /usr/local/lib/python3.10/dist-packages (from axolotl[deepspeed]) (1.3)\n",
            "Requirement already satisfied: torch==2.5.1+cu121 in /usr/local/lib/python3.10/dist-packages (from axolotl[deepspeed]) (2.5.1+cu121)\n",
            "Requirement already satisfied: xformers==0.0.28.post3 in /usr/local/lib/python3.10/dist-packages (from axolotl[deepspeed]) (0.0.28.post3)\n",
            "Requirement already satisfied: deepspeed==0.16.1 in /usr/local/lib/python3.10/dist-packages (from axolotl[deepspeed]) (0.16.1)\n",
            "Requirement already satisfied: deepspeed-kernels in /usr/local/lib/python3.10/dist-packages (from axolotl[deepspeed]) (0.0.1.dev1698255861)\n",
            "Requirement already satisfied: psutil in /usr/local/lib/python3.10/dist-packages (from accelerate==1.2.0->axolotl[deepspeed]) (5.9.5)\n",
            "Requirement already satisfied: huggingface-hub>=0.21.0 in /usr/local/lib/python3.10/dist-packages (from accelerate==1.2.0->axolotl[deepspeed]) (0.27.0)\n",
            "Requirement already satisfied: safetensors>=0.4.3 in /usr/local/lib/python3.10/dist-packages (from accelerate==1.2.0->axolotl[deepspeed]) (0.4.5)\n",
            "Requirement already satisfied: typing_extensions>=4.8.0 in /usr/local/lib/python3.10/dist-packages (from bitsandbytes==0.45.0->axolotl[deepspeed]) (4.12.2)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from datasets==3.1.0->axolotl[deepspeed]) (3.16.1)\n",
            "Requirement already satisfied: pyarrow>=15.0.0 in /usr/local/lib/python3.10/dist-packages (from datasets==3.1.0->axolotl[deepspeed]) (17.0.0)\n",
            "Requirement already satisfied: dill<0.3.9,>=0.3.0 in /usr/local/lib/python3.10/dist-packages (from datasets==3.1.0->axolotl[deepspeed]) (0.3.8)\n",
            "Requirement already satisfied: pandas in /usr/local/lib/python3.10/dist-packages (from datasets==3.1.0->axolotl[deepspeed]) (2.2.2)\n",
            "Requirement already satisfied: tqdm>=4.66.3 in /usr/local/lib/python3.10/dist-packages (from datasets==3.1.0->axolotl[deepspeed]) (4.67.1)\n",
            "Requirement already satisfied: xxhash in /usr/local/lib/python3.10/dist-packages (from datasets==3.1.0->axolotl[deepspeed]) (3.5.0)\n",
            "Requirement already satisfied: multiprocess<0.70.17 in /usr/local/lib/python3.10/dist-packages (from datasets==3.1.0->axolotl[deepspeed]) (0.70.16)\n",
            "Requirement already satisfied: fsspec<=2024.9.0,>=2023.1.0 in /usr/local/lib/python3.10/dist-packages (from fsspec[http]<=2024.9.0,>=2023.1.0->datasets==3.1.0->axolotl[deepspeed]) (2024.9.0)\n",
            "Requirement already satisfied: aiohttp in /usr/local/lib/python3.10/dist-packages (from datasets==3.1.0->axolotl[deepspeed]) (3.11.10)\n",
            "Requirement already satisfied: hjson in /usr/local/lib/python3.10/dist-packages (from deepspeed==0.16.1->axolotl[deepspeed]) (3.1.0)\n",
            "Requirement already satisfied: msgpack in /usr/local/lib/python3.10/dist-packages (from deepspeed==0.16.1->axolotl[deepspeed]) (1.1.0)\n",
            "Requirement already satisfied: ninja in /usr/local/lib/python3.10/dist-packages (from deepspeed==0.16.1->axolotl[deepspeed]) (1.11.1.3)\n",
            "Requirement already satisfied: py-cpuinfo in /usr/local/lib/python3.10/dist-packages (from deepspeed==0.16.1->axolotl[deepspeed]) (9.0.0)\n",
            "Requirement already satisfied: responses<0.19 in /usr/local/lib/python3.10/dist-packages (from evaluate==0.4.1->axolotl[deepspeed]) (0.18.0)\n",
            "Requirement already satisfied: aiofiles<24.0,>=22.0 in /usr/local/lib/python3.10/dist-packages (from gradio==3.50.2->axolotl[deepspeed]) (23.2.1)\n",
            "Requirement already satisfied: altair<6.0,>=4.2.0 in /usr/local/lib/python3.10/dist-packages (from gradio==3.50.2->axolotl[deepspeed]) (5.5.0)\n",
            "Requirement already satisfied: fastapi in /usr/local/lib/python3.10/dist-packages (from gradio==3.50.2->axolotl[deepspeed]) (0.115.6)\n",
            "Requirement already satisfied: ffmpy in /usr/local/lib/python3.10/dist-packages (from gradio==3.50.2->axolotl[deepspeed]) (0.5.0)\n",
            "Requirement already satisfied: gradio-client==0.6.1 in /usr/local/lib/python3.10/dist-packages (from gradio==3.50.2->axolotl[deepspeed]) (0.6.1)\n",
            "Requirement already satisfied: httpx in /usr/local/lib/python3.10/dist-packages (from gradio==3.50.2->axolotl[deepspeed]) (0.28.1)\n",
            "Requirement already satisfied: importlib-resources<7.0,>=1.3 in /usr/local/lib/python3.10/dist-packages (from gradio==3.50.2->axolotl[deepspeed]) (6.4.5)\n",
            "Requirement already satisfied: jinja2<4.0 in /usr/local/lib/python3.10/dist-packages (from gradio==3.50.2->axolotl[deepspeed]) (3.1.4)\n",
            "Requirement already satisfied: markupsafe~=2.0 in /usr/local/lib/python3.10/dist-packages (from gradio==3.50.2->axolotl[deepspeed]) (2.1.5)\n",
            "Requirement already satisfied: matplotlib~=3.0 in /usr/local/lib/python3.10/dist-packages (from gradio==3.50.2->axolotl[deepspeed]) (3.8.0)\n",
            "Requirement already satisfied: orjson~=3.0 in /usr/local/lib/python3.10/dist-packages (from gradio==3.50.2->axolotl[deepspeed]) (3.10.12)\n",
            "Requirement already satisfied: pillow<11.0,>=8.0 in /usr/local/lib/python3.10/dist-packages (from gradio==3.50.2->axolotl[deepspeed]) (10.4.0)\n",
            "Requirement already satisfied: pydub in /usr/local/lib/python3.10/dist-packages (from gradio==3.50.2->axolotl[deepspeed]) (0.25.1)\n",
            "Requirement already satisfied: python-multipart in /usr/local/lib/python3.10/dist-packages (from gradio==3.50.2->axolotl[deepspeed]) (0.0.20)\n",
            "Requirement already satisfied: semantic-version~=2.0 in /usr/local/lib/python3.10/dist-packages (from gradio==3.50.2->axolotl[deepspeed]) (2.10.0)\n",
            "Requirement already satisfied: uvicorn>=0.14.0 in /usr/local/lib/python3.10/dist-packages (from gradio==3.50.2->axolotl[deepspeed]) (0.34.0)\n",
            "Requirement already satisfied: websockets<12.0,>=10.0 in /usr/local/lib/python3.10/dist-packages (from gradio==3.50.2->axolotl[deepspeed]) (11.0.3)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.10/dist-packages (from langdetect==1.0.9->axolotl[deepspeed]) (1.17.0)\n",
            "Requirement already satisfied: jsonlines in /usr/local/lib/python3.10/dist-packages (from lm-eval==0.4.4->axolotl[deepspeed]) (4.0.0)\n",
            "Requirement already satisfied: numexpr in /usr/local/lib/python3.10/dist-packages (from lm-eval==0.4.4->axolotl[deepspeed]) (2.10.2)\n",
            "Requirement already satisfied: pybind11>=2.6.2 in /usr/local/lib/python3.10/dist-packages (from lm-eval==0.4.4->axolotl[deepspeed]) (2.13.6)\n",
            "Requirement already satisfied: pytablewriter in /usr/local/lib/python3.10/dist-packages (from lm-eval==0.4.4->axolotl[deepspeed]) (1.2.1)\n",
            "Requirement already satisfied: rouge-score>=0.0.4 in /usr/local/lib/python3.10/dist-packages (from lm-eval==0.4.4->axolotl[deepspeed]) (0.1.2)\n",
            "Requirement already satisfied: sacrebleu>=1.5.0 in /usr/local/lib/python3.10/dist-packages (from lm-eval==0.4.4->axolotl[deepspeed]) (2.4.3)\n",
            "Requirement already satisfied: sqlitedict in /usr/local/lib/python3.10/dist-packages (from lm-eval==0.4.4->axolotl[deepspeed]) (2.1.0)\n",
            "Requirement already satisfied: tqdm-multiprocess in /usr/local/lib/python3.10/dist-packages (from lm-eval==0.4.4->axolotl[deepspeed]) (0.0.11)\n",
            "Requirement already satisfied: word2number in /usr/local/lib/python3.10/dist-packages (from lm-eval==0.4.4->axolotl[deepspeed]) (1.1)\n",
            "Requirement already satisfied: more-itertools in /usr/local/lib/python3.10/dist-packages (from lm-eval==0.4.4->axolotl[deepspeed]) (10.5.0)\n",
            "Requirement already satisfied: coloredlogs in /usr/local/lib/python3.10/dist-packages (from optimum==1.16.2->axolotl[deepspeed]) (15.0.1)\n",
            "Requirement already satisfied: sympy in /usr/local/lib/python3.10/dist-packages (from optimum==1.16.2->axolotl[deepspeed]) (1.13.1)\n",
            "Requirement already satisfied: annotated-types>=0.4.0 in /usr/local/lib/python3.10/dist-packages (from pydantic==2.6.3->axolotl[deepspeed]) (0.7.0)\n",
            "Requirement already satisfied: pydantic-core==2.16.3 in /usr/local/lib/python3.10/dist-packages (from pydantic==2.6.3->axolotl[deepspeed]) (2.16.3)\n",
            "Requirement already satisfied: joblib>=1.2.0 in /usr/local/lib/python3.10/dist-packages (from scikit-learn==1.4.2->axolotl[deepspeed]) (1.4.2)\n",
            "Requirement already satisfied: threadpoolctl>=2.0.0 in /usr/local/lib/python3.10/dist-packages (from scikit-learn==1.4.2->axolotl[deepspeed]) (3.5.0)\n",
            "Requirement already satisfied: networkx in /usr/local/lib/python3.10/dist-packages (from torch==2.5.1+cu121->axolotl[deepspeed]) (3.4.2)\n",
            "Requirement already satisfied: rich in /usr/local/lib/python3.10/dist-packages (from trl==0.12.1->axolotl[deepspeed]) (13.9.4)\n",
            "Requirement already satisfied: mpmath<1.4,>=1.1.0 in /usr/local/lib/python3.10/dist-packages (from sympy->optimum==1.16.2->axolotl[deepspeed]) (1.3.0)\n",
            "Requirement already satisfied: decorator>4.1.2 in /usr/local/lib/python3.10/dist-packages (from gcsfs>=2024.5.0->axolotl[deepspeed]) (4.4.2)\n",
            "Requirement already satisfied: google-auth>=1.2 in /usr/local/lib/python3.10/dist-packages (from gcsfs>=2024.5.0->axolotl[deepspeed]) (2.27.0)\n",
            "Requirement already satisfied: google-auth-oauthlib in /usr/local/lib/python3.10/dist-packages (from gcsfs>=2024.5.0->axolotl[deepspeed]) (1.2.1)\n",
            "Requirement already satisfied: google-cloud-storage in /usr/local/lib/python3.10/dist-packages (from gcsfs>=2024.5.0->axolotl[deepspeed]) (2.19.0)\n",
            "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests->axolotl[deepspeed]) (3.4.0)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests->axolotl[deepspeed]) (3.10)\n",
            "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests->axolotl[deepspeed]) (2.2.3)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests->axolotl[deepspeed]) (2024.12.14)\n",
            "Requirement already satisfied: aiobotocore<3.0.0,>=2.5.4 in /usr/local/lib/python3.10/dist-packages (from s3fs>=2024.5.0->axolotl[deepspeed]) (2.16.1)\n",
            "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.10/dist-packages (from transformers>=4.46.3->axolotl[deepspeed]) (2024.11.6)\n",
            "Requirement already satisfied: cmake>=3.24 in /usr/local/lib/python3.10/dist-packages (from deepspeed-kernels->axolotl[deepspeed]) (3.31.2)\n",
            "Requirement already satisfied: termcolor in /usr/local/lib/python3.10/dist-packages (from fire->axolotl[deepspeed]) (2.5.0)\n",
            "Requirement already satisfied: llvmlite<0.44,>=0.43.0dev0 in /usr/local/lib/python3.10/dist-packages (from numba->axolotl[deepspeed]) (0.43.0)\n",
            "Requirement already satisfied: absl-py>=0.4 in /usr/local/lib/python3.10/dist-packages (from tensorboard->axolotl[deepspeed]) (1.4.0)\n",
            "Requirement already satisfied: grpcio>=1.48.2 in /usr/local/lib/python3.10/dist-packages (from tensorboard->axolotl[deepspeed]) (1.68.1)\n",
            "Requirement already satisfied: markdown>=2.6.8 in /usr/local/lib/python3.10/dist-packages (from tensorboard->axolotl[deepspeed]) (3.7)\n",
            "Requirement already satisfied: protobuf!=4.24.0,>=3.19.6 in /usr/local/lib/python3.10/dist-packages (from tensorboard->axolotl[deepspeed]) (4.25.5)\n",
            "Requirement already satisfied: setuptools>=41.0.0 in /usr/local/lib/python3.10/dist-packages (from tensorboard->axolotl[deepspeed]) (75.1.0)\n",
            "Requirement already satisfied: tensorboard-data-server<0.8.0,>=0.7.0 in /usr/local/lib/python3.10/dist-packages (from tensorboard->axolotl[deepspeed]) (0.7.2)\n",
            "Requirement already satisfied: werkzeug>=1.0.1 in /usr/local/lib/python3.10/dist-packages (from tensorboard->axolotl[deepspeed]) (3.1.3)\n",
            "Requirement already satisfied: click!=8.0.0,>=7.1 in /usr/local/lib/python3.10/dist-packages (from wandb->axolotl[deepspeed]) (8.1.7)\n",
            "Requirement already satisfied: docker-pycreds>=0.4.0 in /usr/local/lib/python3.10/dist-packages (from wandb->axolotl[deepspeed]) (0.4.0)\n",
            "Requirement already satisfied: gitpython!=3.1.29,>=1.0.0 in /usr/local/lib/python3.10/dist-packages (from wandb->axolotl[deepspeed]) (3.1.43)\n",
            "Requirement already satisfied: platformdirs in /usr/local/lib/python3.10/dist-packages (from wandb->axolotl[deepspeed]) (4.3.6)\n",
            "Requirement already satisfied: sentry-sdk>=2.0.0 in /usr/local/lib/python3.10/dist-packages (from wandb->axolotl[deepspeed]) (2.19.2)\n",
            "Requirement already satisfied: setproctitle in /usr/local/lib/python3.10/dist-packages (from wandb->axolotl[deepspeed]) (1.3.4)\n",
            "Requirement already satisfied: botocore<1.35.89,>=1.35.74 in /usr/local/lib/python3.10/dist-packages (from aiobotocore<3.0.0,>=2.5.4->s3fs>=2024.5.0->axolotl[deepspeed]) (1.35.88)\n",
            "Requirement already satisfied: wrapt<2.0.0,>=1.10.10 in /usr/local/lib/python3.10/dist-packages (from aiobotocore<3.0.0,>=2.5.4->s3fs>=2024.5.0->axolotl[deepspeed]) (1.17.0)\n",
            "Requirement already satisfied: aioitertools<1.0.0,>=0.5.1 in /usr/local/lib/python3.10/dist-packages (from aiobotocore<3.0.0,>=2.5.4->s3fs>=2024.5.0->axolotl[deepspeed]) (0.12.0)\n",
            "Requirement already satisfied: aiohappyeyeballs>=2.3.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets==3.1.0->axolotl[deepspeed]) (2.4.4)\n",
            "Requirement already satisfied: aiosignal>=1.1.2 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets==3.1.0->axolotl[deepspeed]) (1.3.2)\n",
            "Requirement already satisfied: async-timeout<6.0,>=4.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets==3.1.0->axolotl[deepspeed]) (4.0.3)\n",
            "Requirement already satisfied: attrs>=17.3.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets==3.1.0->axolotl[deepspeed]) (24.3.0)\n",
            "Requirement already satisfied: frozenlist>=1.1.1 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets==3.1.0->axolotl[deepspeed]) (1.5.0)\n",
            "Requirement already satisfied: multidict<7.0,>=4.5 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets==3.1.0->axolotl[deepspeed]) (6.1.0)\n",
            "Requirement already satisfied: propcache>=0.2.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets==3.1.0->axolotl[deepspeed]) (0.2.1)\n",
            "Requirement already satisfied: yarl<2.0,>=1.17.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets==3.1.0->axolotl[deepspeed]) (1.18.3)\n",
            "Requirement already satisfied: jsonschema>=3.0 in /usr/local/lib/python3.10/dist-packages (from altair<6.0,>=4.2.0->gradio==3.50.2->axolotl[deepspeed]) (4.23.0)\n",
            "Requirement already satisfied: narwhals>=1.14.2 in /usr/local/lib/python3.10/dist-packages (from altair<6.0,>=4.2.0->gradio==3.50.2->axolotl[deepspeed]) (1.18.4)\n",
            "Requirement already satisfied: gitdb<5,>=4.0.1 in /usr/local/lib/python3.10/dist-packages (from gitpython!=3.1.29,>=1.0.0->wandb->axolotl[deepspeed]) (4.0.11)\n",
            "Requirement already satisfied: cachetools<6.0,>=2.0.0 in /usr/local/lib/python3.10/dist-packages (from google-auth>=1.2->gcsfs>=2024.5.0->axolotl[deepspeed]) (5.5.0)\n",
            "Requirement already satisfied: pyasn1-modules>=0.2.1 in /usr/local/lib/python3.10/dist-packages (from google-auth>=1.2->gcsfs>=2024.5.0->axolotl[deepspeed]) (0.4.1)\n",
            "Requirement already satisfied: rsa<5,>=3.1.4 in /usr/local/lib/python3.10/dist-packages (from google-auth>=1.2->gcsfs>=2024.5.0->axolotl[deepspeed]) (4.9)\n",
            "Requirement already satisfied: contourpy>=1.0.1 in /usr/local/lib/python3.10/dist-packages (from matplotlib~=3.0->gradio==3.50.2->axolotl[deepspeed]) (1.3.1)\n",
            "Requirement already satisfied: cycler>=0.10 in /usr/local/lib/python3.10/dist-packages (from matplotlib~=3.0->gradio==3.50.2->axolotl[deepspeed]) (0.12.1)\n",
            "Requirement already satisfied: fonttools>=4.22.0 in /usr/local/lib/python3.10/dist-packages (from matplotlib~=3.0->gradio==3.50.2->axolotl[deepspeed]) (4.55.3)\n",
            "Requirement already satisfied: kiwisolver>=1.0.1 in /usr/local/lib/python3.10/dist-packages (from matplotlib~=3.0->gradio==3.50.2->axolotl[deepspeed]) (1.4.7)\n",
            "Requirement already satisfied: pyparsing>=2.3.1 in /usr/local/lib/python3.10/dist-packages (from matplotlib~=3.0->gradio==3.50.2->axolotl[deepspeed]) (3.2.0)\n",
            "Requirement already satisfied: python-dateutil>=2.7 in /usr/local/lib/python3.10/dist-packages (from matplotlib~=3.0->gradio==3.50.2->axolotl[deepspeed]) (2.8.2)\n",
            "Requirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.10/dist-packages (from pandas->datasets==3.1.0->axolotl[deepspeed]) (2024.2)\n",
            "Requirement already satisfied: tzdata>=2022.7 in /usr/local/lib/python3.10/dist-packages (from pandas->datasets==3.1.0->axolotl[deepspeed]) (2024.2)\n",
            "Requirement already satisfied: nltk in /usr/local/lib/python3.10/dist-packages (from rouge-score>=0.0.4->lm-eval==0.4.4->axolotl[deepspeed]) (3.9.1)\n",
            "Requirement already satisfied: portalocker in /usr/local/lib/python3.10/dist-packages (from sacrebleu>=1.5.0->lm-eval==0.4.4->axolotl[deepspeed]) (3.1.1)\n",
            "Requirement already satisfied: tabulate>=0.8.9 in /usr/local/lib/python3.10/dist-packages (from sacrebleu>=1.5.0->lm-eval==0.4.4->axolotl[deepspeed]) (0.9.0)\n",
            "Requirement already satisfied: lxml in /usr/local/lib/python3.10/dist-packages (from sacrebleu>=1.5.0->lm-eval==0.4.4->axolotl[deepspeed]) (5.3.0)\n",
            "Requirement already satisfied: h11>=0.8 in /usr/local/lib/python3.10/dist-packages (from uvicorn>=0.14.0->gradio==3.50.2->axolotl[deepspeed]) (0.14.0)\n",
            "Requirement already satisfied: humanfriendly>=9.1 in /usr/local/lib/python3.10/dist-packages (from coloredlogs->optimum==1.16.2->axolotl[deepspeed]) (10.0)\n",
            "Requirement already satisfied: starlette<0.42.0,>=0.40.0 in /usr/local/lib/python3.10/dist-packages (from fastapi->gradio==3.50.2->axolotl[deepspeed]) (0.41.3)\n",
            "Requirement already satisfied: requests-oauthlib>=0.7.0 in /usr/local/lib/python3.10/dist-packages (from google-auth-oauthlib->gcsfs>=2024.5.0->axolotl[deepspeed]) (1.3.1)\n",
            "Requirement already satisfied: google-api-core<3.0.0dev,>=2.15.0 in /usr/local/lib/python3.10/dist-packages (from google-cloud-storage->gcsfs>=2024.5.0->axolotl[deepspeed]) (2.19.2)\n",
            "Requirement already satisfied: google-cloud-core<3.0dev,>=2.3.0 in /usr/local/lib/python3.10/dist-packages (from google-cloud-storage->gcsfs>=2024.5.0->axolotl[deepspeed]) (2.4.1)\n",
            "Requirement already satisfied: google-resumable-media>=2.7.2 in /usr/local/lib/python3.10/dist-packages (from google-cloud-storage->gcsfs>=2024.5.0->axolotl[deepspeed]) (2.7.2)\n",
            "Requirement already satisfied: google-crc32c<2.0dev,>=1.0 in /usr/local/lib/python3.10/dist-packages (from google-cloud-storage->gcsfs>=2024.5.0->axolotl[deepspeed]) (1.6.0)\n",
            "Requirement already satisfied: anyio in /usr/local/lib/python3.10/dist-packages (from httpx->gradio==3.50.2->axolotl[deepspeed]) (3.7.1)\n",
            "Requirement already satisfied: httpcore==1.* in /usr/local/lib/python3.10/dist-packages (from httpx->gradio==3.50.2->axolotl[deepspeed]) (1.0.7)\n",
            "Requirement already satisfied: DataProperty<2,>=1.1.0 in /usr/local/lib/python3.10/dist-packages (from pytablewriter->lm-eval==0.4.4->axolotl[deepspeed]) (1.1.0)\n",
            "Requirement already satisfied: mbstrdecoder<2,>=1.0.0 in /usr/local/lib/python3.10/dist-packages (from pytablewriter->lm-eval==0.4.4->axolotl[deepspeed]) (1.1.3)\n",
            "Requirement already satisfied: pathvalidate<4,>=2.3.0 in /usr/local/lib/python3.10/dist-packages (from pytablewriter->lm-eval==0.4.4->axolotl[deepspeed]) (3.2.2)\n",
            "Requirement already satisfied: tabledata<2,>=1.3.1 in /usr/local/lib/python3.10/dist-packages (from pytablewriter->lm-eval==0.4.4->axolotl[deepspeed]) (1.3.4)\n",
            "Requirement already satisfied: tcolorpy<1,>=0.0.5 in /usr/local/lib/python3.10/dist-packages (from pytablewriter->lm-eval==0.4.4->axolotl[deepspeed]) (0.1.7)\n",
            "Requirement already satisfied: typepy<2,>=1.3.2 in /usr/local/lib/python3.10/dist-packages (from typepy[datetime]<2,>=1.3.2->pytablewriter->lm-eval==0.4.4->axolotl[deepspeed]) (1.3.4)\n",
            "Requirement already satisfied: markdown-it-py>=2.2.0 in /usr/local/lib/python3.10/dist-packages (from rich->trl==0.12.1->axolotl[deepspeed]) (3.0.0)\n",
            "Requirement already satisfied: pygments<3.0.0,>=2.13.0 in /usr/local/lib/python3.10/dist-packages (from rich->trl==0.12.1->axolotl[deepspeed]) (2.18.0)\n",
            "Requirement already satisfied: jmespath<2.0.0,>=0.7.1 in /usr/local/lib/python3.10/dist-packages (from botocore<1.35.89,>=1.35.74->aiobotocore<3.0.0,>=2.5.4->s3fs>=2024.5.0->axolotl[deepspeed]) (1.0.1)\n",
            "Requirement already satisfied: smmap<6,>=3.0.1 in /usr/local/lib/python3.10/dist-packages (from gitdb<5,>=4.0.1->gitpython!=3.1.29,>=1.0.0->wandb->axolotl[deepspeed]) (5.0.1)\n",
            "Requirement already satisfied: googleapis-common-protos<2.0.dev0,>=1.56.2 in /usr/local/lib/python3.10/dist-packages (from google-api-core<3.0.0dev,>=2.15.0->google-cloud-storage->gcsfs>=2024.5.0->axolotl[deepspeed]) (1.66.0)\n",
            "Requirement already satisfied: proto-plus<2.0.0dev,>=1.22.3 in /usr/local/lib/python3.10/dist-packages (from google-api-core<3.0.0dev,>=2.15.0->google-cloud-storage->gcsfs>=2024.5.0->axolotl[deepspeed]) (1.25.0)\n",
            "Requirement already satisfied: jsonschema-specifications>=2023.03.6 in /usr/local/lib/python3.10/dist-packages (from jsonschema>=3.0->altair<6.0,>=4.2.0->gradio==3.50.2->axolotl[deepspeed]) (2024.10.1)\n",
            "Requirement already satisfied: referencing>=0.28.4 in /usr/local/lib/python3.10/dist-packages (from jsonschema>=3.0->altair<6.0,>=4.2.0->gradio==3.50.2->axolotl[deepspeed]) (0.35.1)\n",
            "Requirement already satisfied: rpds-py>=0.7.1 in /usr/local/lib/python3.10/dist-packages (from jsonschema>=3.0->altair<6.0,>=4.2.0->gradio==3.50.2->axolotl[deepspeed]) (0.22.3)\n",
            "Requirement already satisfied: mdurl~=0.1 in /usr/local/lib/python3.10/dist-packages (from markdown-it-py>=2.2.0->rich->trl==0.12.1->axolotl[deepspeed]) (0.1.2)\n",
            "Requirement already satisfied: chardet<6,>=3.0.4 in /usr/local/lib/python3.10/dist-packages (from mbstrdecoder<2,>=1.0.0->pytablewriter->lm-eval==0.4.4->axolotl[deepspeed]) (5.2.0)\n",
            "Requirement already satisfied: pyasn1<0.7.0,>=0.4.6 in /usr/local/lib/python3.10/dist-packages (from pyasn1-modules>=0.2.1->google-auth>=1.2->gcsfs>=2024.5.0->axolotl[deepspeed]) (0.6.1)\n",
            "Requirement already satisfied: oauthlib>=3.0.0 in /usr/local/lib/python3.10/dist-packages (from requests-oauthlib>=0.7.0->google-auth-oauthlib->gcsfs>=2024.5.0->axolotl[deepspeed]) (3.2.2)\n",
            "Requirement already satisfied: sniffio>=1.1 in /usr/local/lib/python3.10/dist-packages (from anyio->httpx->gradio==3.50.2->axolotl[deepspeed]) (1.3.1)\n",
            "Requirement already satisfied: exceptiongroup in /usr/local/lib/python3.10/dist-packages (from anyio->httpx->gradio==3.50.2->axolotl[deepspeed]) (1.2.2)\n"
          ]
        }
      ],
      "source": [
        "!pip install --no-build-isolation axolotl[deepspeed]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 3,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 17,
          "referenced_widgets": [
            "b106e1468b7d4b1884d43c00d67fbcc5",
            "05e8461f56c04927b7550d323ec8983f",
            "7d065ea3a7e94cfbb552681eb8fb08a9",
            "2bd3022ee06f41f5b4aa02f8bd3bb29a",
            "5d217568638d4f1895170a29856e0db8",
            "2aa1012f20274906b534d9e11f868f74",
            "d7e7e44345ea447e9f67fe97eeec0bcb",
            "7a7b319056cb4839992465c4a8e12b33",
            "7e9cbdc5ccc3408f9af8398f8cffdca1",
            "8032f2a7e1774260841468647cad3cc9",
            "2365fbc6e3304b2ea845c8c5b7c0ee9b",
            "b399b473b43b4abfb5ae1ad0af2fb691",
            "4500924c0fc243af9edc9d294e698f9a",
            "bd1910d24b9740f88a3c1b02d2360a0d",
            "fce095a07d0f4b4faf1ff15f351e8828",
            "6406cc0b52c54d63bbba9768259c4dfa",
            "25bfa563cb4c452487d48212b821c503",
            "be1dbf5ef425448aa9f0b5b2a2d771c0",
            "1c619dc43d7447788b3a11863f67283a",
            "7016d2e7cf734b9e8ee09b39c96ae483"
          ]
        },
        "id": "uHdqJsmrAG9P",
        "outputId": "c825a4b7-4b97-43da-a9d7-96047d359d42"
      },
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "VBox(children=(HTML(value='<center> <img\\nsrc=https://huggingface.co/front/assets/huggingface_logo-noborder.sv…"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "b106e1468b7d4b1884d43c00d67fbcc5"
            }
          },
          "metadata": {}
        }
      ],
      "source": [
        "from huggingface_hub import notebook_login\n",
        "notebook_login()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 4,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 138
        },
        "id": "WeXG7671C4o8",
        "outputId": "583b921a-6747-4a03-870c-9507044e8306"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\u001b[34m\u001b[1mwandb\u001b[0m: Using wandb-core as the SDK backend.  Please refer to https://wandb.me/wandb-core for more information.\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.Javascript object>"
            ],
            "application/javascript": [
              "\n",
              "        window._wandbApiKey = new Promise((resolve, reject) => {\n",
              "            function loadScript(url) {\n",
              "            return new Promise(function(resolve, reject) {\n",
              "                let newScript = document.createElement(\"script\");\n",
              "                newScript.onerror = reject;\n",
              "                newScript.onload = resolve;\n",
              "                document.body.appendChild(newScript);\n",
              "                newScript.src = url;\n",
              "            });\n",
              "            }\n",
              "            loadScript(\"https://cdn.jsdelivr.net/npm/postmate/build/postmate.min.js\").then(() => {\n",
              "            const iframe = document.createElement('iframe')\n",
              "            iframe.style.cssText = \"width:0;height:0;border:none\"\n",
              "            document.body.appendChild(iframe)\n",
              "            const handshake = new Postmate({\n",
              "                container: iframe,\n",
              "                url: 'https://wandb.ai/authorize'\n",
              "            });\n",
              "            const timeout = setTimeout(() => reject(\"Couldn't auto authenticate\"), 5000)\n",
              "            handshake.then(function(child) {\n",
              "                child.on('authorize', data => {\n",
              "                    clearTimeout(timeout)\n",
              "                    resolve(data)\n",
              "                });\n",
              "            });\n",
              "            })\n",
              "        });\n",
              "    "
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\u001b[34m\u001b[1mwandb\u001b[0m: Logging into wandb.ai. (Learn how to deploy a W&B server locally: https://wandb.me/wandb-server)\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: You can find your API key in your browser here: https://wandb.ai/authorize\n",
            "wandb: Paste an API key from your profile and hit enter, or press ctrl+c to quit:"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            " ··········\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\u001b[34m\u001b[1mwandb\u001b[0m: Appending key for api.wandb.ai to your netrc file: /root/.netrc\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "True"
            ]
          },
          "metadata": {},
          "execution_count": 4
        }
      ],
      "source": [
        "import wandb\n",
        "wandb.login()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 5,
      "metadata": {
        "id": "ymDBg46TAG9Q"
      },
      "outputs": [],
      "source": [
        "import yaml\n",
        "\n",
        "yaml_string = \"\"\"\n",
        "base_model: TinyLlama/TinyLlama_v1.1\n",
        "\n",
        "model_type: LlamaForCausalLM\n",
        "tokenizer_type: LlamaTokenizer\n",
        "\n",
        "hub_model_id: EvolCodeTinyLlama\n",
        "\n",
        "load_in_8bit: false\n",
        "load_in_4bit: true\n",
        "strict: false\n",
        "\n",
        "datasets:\n",
        "    - path: mlabonne/Evol-Instruct-Python-1k\n",
        "      type: alpaca\n",
        "dataset_prepared_path: last_run_prepared\n",
        "val_set_size: 0.02\n",
        "output_dir: ./qlora-out\n",
        "\n",
        "adapter: qlora\n",
        "lora_model_dir:\n",
        "\n",
        "sequence_len: 2048\n",
        "sample_packing: true\n",
        "eval_sample_packing: False\n",
        "\n",
        "lora_r: 32\n",
        "lora_alpha: 16\n",
        "lora_dropout: 0.05\n",
        "lora_target_modules:\n",
        "lora_target_linear: true\n",
        "lora_fan_in_fan_out:\n",
        "\n",
        "wandb_project: axolotl\n",
        "wandb_entity:\n",
        "wandb_watch:\n",
        "wandb_run_id:\n",
        "wandb_log_model:\n",
        "\n",
        "gradient_accumulation_steps: 2\n",
        "micro_batch_size: 2\n",
        "num_epochs: 3\n",
        "optimizer: paged_adamw_32bit\n",
        "lr_scheduler: cosine\n",
        "learning_rate: 0.0002\n",
        "\n",
        "train_on_inputs: false\n",
        "group_by_length: false\n",
        "bf16: true\n",
        "fp16: false\n",
        "tf32: false\n",
        "\n",
        "gradient_checkpointing: true\n",
        "early_stopping_patience:\n",
        "resume_from_checkpoint:\n",
        "local_rank:\n",
        "logging_steps: 1\n",
        "xformers_attention:\n",
        "flash_attention: false\n",
        "\n",
        "warmup_steps: 10\n",
        "eval_steps: 0.01\n",
        "save_strategy: epoch\n",
        "save_steps:\n",
        "debug:\n",
        "deepspeed:\n",
        "weight_decay: 0.0\n",
        "fsdp:\n",
        "fsdp_config:\n",
        "special_tokens:\n",
        "\"\"\"\n",
        "\n",
        "\n",
        "# Convert the YAML string to a Python dictionary\n",
        "yaml_dict = yaml.safe_load(yaml_string)\n",
        "\n",
        "# Specify your file path\n",
        "file_path = 'tinyllama.yaml'\n",
        "\n",
        "# Write the YAML file\n",
        "with open(file_path, 'w') as file:\n",
        "    yaml.dump(yaml_dict, file)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "yoZl_IBNAG9R"
      },
      "source": [
        "Train the model"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "X22F5qNxAG9R",
        "outputId": "1e5205ef-6ab0-435a-d598-6ca2171a2c84"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "The following values were not passed to `accelerate launch` and had defaults used instead:\n",
            "\t`--num_processes` was set to a value of `1`\n",
            "\t`--num_machines` was set to a value of `1`\n",
            "\t`--mixed_precision` was set to a value of `'no'`\n",
            "\t`--dynamo_backend` was set to a value of `'no'`\n",
            "To avoid this warning pass in values for each of the problematic parameters or run `accelerate config`.\n",
            "2025-01-02 12:58:24.842111: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:485] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered\n",
            "2025-01-02 12:58:24.861655: E external/local_xla/xla/stream_executor/cuda/cuda_dnn.cc:8454] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered\n",
            "2025-01-02 12:58:24.869345: E external/local_xla/xla/stream_executor/cuda/cuda_blas.cc:1452] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered\n",
            "2025-01-02 12:58:24.884110: I tensorflow/core/platform/cpu_feature_guard.cc:210] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations.\n",
            "To enable the following instructions: AVX2 AVX512F FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags.\n",
            "2025-01-02 12:58:25.889117: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT\n",
            "[2025-01-02 12:58:28,330] [INFO] [datasets.<module>:54] [PID:2106] PyTorch version 2.5.1+cu121 available.\n",
            "[2025-01-02 12:58:28,332] [INFO] [datasets.<module>:66] [PID:2106] Polars version 1.9.0 available.\n",
            "[2025-01-02 12:58:28,334] [INFO] [datasets.<module>:77] [PID:2106] Duckdb version 1.1.3 available.\n",
            "[2025-01-02 12:58:28,334] [INFO] [datasets.<module>:112] [PID:2106] TensorFlow version 2.17.1 available.\n",
            "[2025-01-02 12:58:28,335] [INFO] [datasets.<module>:125] [PID:2106] JAX version 0.4.33 available.\n",
            "[2025-01-02 12:58:30,599] [INFO] [real_accelerator.py:219:get_accelerator] Setting ds_accelerator to cuda (auto detect)\n",
            "df: /root/.triton/autotune: No such file or directory\n",
            "[2025-01-02 12:58:30,691] [INFO] [root.spawn:60] [PID:2106] x86_64-linux-gnu-gcc -Wno-unused-result -Wsign-compare -DNDEBUG -g -fwrapv -O2 -Wall -g -fstack-protector-strong -Wformat -Werror=format-security -g -fwrapv -O2 -fPIC -c /tmp/tmpkanmc52j/test.c -o /tmp/tmpkanmc52j/test.o\n",
            "[2025-01-02 12:58:30,708] [INFO] [root.spawn:60] [PID:2106] x86_64-linux-gnu-gcc /tmp/tmpkanmc52j/test.o -laio -o /tmp/tmpkanmc52j/a.out\n",
            "[2025-01-02 12:58:31,346] [INFO] [root.spawn:60] [PID:2106] x86_64-linux-gnu-gcc -Wno-unused-result -Wsign-compare -DNDEBUG -g -fwrapv -O2 -Wall -g -fstack-protector-strong -Wformat -Werror=format-security -g -fwrapv -O2 -fPIC -c /tmp/tmpwxro5zpc/test.c -o /tmp/tmpwxro5zpc/test.o\n",
            "[2025-01-02 12:58:31,362] [INFO] [root.spawn:60] [PID:2106] x86_64-linux-gnu-gcc /tmp/tmpwxro5zpc/test.o -L/usr/local/cuda -L/usr/local/cuda/lib64 -lcufile -o /tmp/tmpwxro5zpc/a.out\n",
            "[2025-01-02 12:58:31,407] [INFO] [root.spawn:60] [PID:2106] x86_64-linux-gnu-gcc -Wno-unused-result -Wsign-compare -DNDEBUG -g -fwrapv -O2 -Wall -g -fstack-protector-strong -Wformat -Werror=format-security -g -fwrapv -O2 -fPIC -c /tmp/tmpcmqopqoz/test.c -o /tmp/tmpcmqopqoz/test.o\n",
            "[2025-01-02 12:58:31,423] [INFO] [root.spawn:60] [PID:2106] x86_64-linux-gnu-gcc /tmp/tmpcmqopqoz/test.o -laio -o /tmp/tmpcmqopqoz/a.out\n",
            "/usr/local/lib/python3.10/dist-packages/axolotl/monkeypatch/relora.py:16: DeprecationWarning: `TorchScript` support for functional optimizers is deprecated and will be removed in a future PyTorch release. Consider using the `torch.compile` optimizer instead.\n",
            "  from torch.distributed.optim import ZeroRedundancyOptimizer\n",
            "/usr/local/lib/python3.10/dist-packages/pydantic/_internal/_fields.py:151: UserWarning: Field \"model_kwargs\" has conflict with protected namespace \"model_\".\n",
            "\n",
            "You may be able to resolve this warning by setting `model_config['protected_namespaces'] = ()`.\n",
            "  warnings.warn(\n",
            "[2025-01-02 12:58:35,332] [INFO] [axolotl.utils.config.models.input.check_eval_packing:1136] [PID:2106] [RANK:0] setting `remove_unused_columns: false` for when sample_packing and eval_sample_packing don't match\u001b[39m\n",
            "\u001b[33m[2025-01-02 12:58:35,332] [WARNING] [axolotl.utils.config.models.input.hint_sample_packing_padding:937] [PID:2106] [RANK:0] `pad_to_sequence_len: true` is recommended when using sample_packing\u001b[39m\n",
            "\u001b[33m[2025-01-02 12:58:35,333] [WARNING] [axolotl.utils.config.models.input.check_sample_packing_wo_flash:920] [PID:2106] [RANK:0] sample_packing without flash_attention or sdp_attention does not handle cross-attention.\u001b[39m\n",
            "config.json: 100% 560/560 [00:00<00:00, 3.54MB/s]\n",
            "[2025-01-02 12:58:35,631] [INFO] [axolotl.normalize_config:211] [PID:2106] [RANK:0] cuda memory usage baseline: 0.000GB (+0.002GB cache, +0.352GB misc)\u001b[39m\n",
            "\n",
            "     #@@ #@@      @@# @@#\n",
            "    @@  @@          @@  @@           =@@#                               @@                 #@    =@@#.\n",
            "    @@    #@@@@@@@@@    @@           #@#@=                              @@                 #@     .=@@\n",
            "      #@@@@@@@@@@@@@@@@@            =@# @#     ##=     ##    =####=+    @@      =#####+  =#@@###.   @@\n",
            "    @@@@@@@@@@/  +@@/  +@@          #@  =@=     #@=   @@   =@#+  +#@#   @@    =@#+  +#@#   #@.      @@\n",
            "    @@@@@@@@@@  ##@@  ##@@         =@#   @#      =@# @#    @@      @@   @@    @@      #@   #@       @@\n",
            "     @@@@@@@@@@@@@@@@@@@@          #@=+++#@=      =@@#     @@      @@   @@    @@      #@   #@       @@\n",
            "                                  =@#=====@@     =@# @#    @@      @@   @@    @@      #@   #@       @@\n",
            "    @@@@@@@@@@@@@@@@  @@@@        #@      #@=   #@=  +@@   #@#    =@#   @@.   =@#    =@#   #@.      @@\n",
            "                                 =@#       @#  #@=     #@   =#@@@@#=    +#@@=  +#@@@@#=    .##@@+   @@\n",
            "    @@@@  @@@@@@@@@@@@@@@@\n",
            "\n",
            "tokenizer_config.json: 100% 776/776 [00:00<00:00, 5.54MB/s]\n",
            "tokenizer.model: 100% 500k/500k [00:00<00:00, 15.5MB/s]\n",
            "special_tokens_map.json: 100% 414/414 [00:00<00:00, 3.33MB/s]\n",
            "tokenizer.json: 100% 1.84M/1.84M [00:00<00:00, 7.41MB/s]\n",
            "[2025-01-02 12:58:37,683] [DEBUG] [axolotl.load_tokenizer:296] [PID:2106] [RANK:0] EOS: 2 / </s>\u001b[39m\n",
            "[2025-01-02 12:58:37,683] [DEBUG] [axolotl.load_tokenizer:297] [PID:2106] [RANK:0] BOS: 1 / <s>\u001b[39m\n",
            "[2025-01-02 12:58:37,684] [DEBUG] [axolotl.load_tokenizer:298] [PID:2106] [RANK:0] PAD: 2 / </s>\u001b[39m\n",
            "[2025-01-02 12:58:37,684] [DEBUG] [axolotl.load_tokenizer:299] [PID:2106] [RANK:0] UNK: 0 / <unk>\u001b[39m\n",
            "[2025-01-02 12:58:37,684] [INFO] [axolotl.load_tokenizer:313] [PID:2106] [RANK:0] No Chat template selected. Consider adding a chat template for easier inference.\u001b[39m\n",
            "[2025-01-02 12:58:37,684] [INFO] [axolotl.load_tokenized_prepared_datasets:216] [PID:2106] [RANK:0] Unable to find prepared dataset in last_run_prepared/d4980cf45b7d886e0a2393e374eb74e7\u001b[39m\n",
            "[2025-01-02 12:58:37,684] [INFO] [axolotl.load_tokenized_prepared_datasets:217] [PID:2106] [RANK:0] Loading raw datasets...\u001b[39m\n",
            "\u001b[33m[2025-01-02 12:58:37,684] [WARNING] [axolotl.load_tokenized_prepared_datasets:219] [PID:2106] [RANK:0] Processing datasets during training can lead to VRAM instability. Please pre-process your dataset.\u001b[39m\n",
            "[2025-01-02 12:58:37,684] [INFO] [axolotl.load_tokenized_prepared_datasets:226] [PID:2106] [RANK:0] No seed provided, using default seed of 42\u001b[39m\n",
            "README.md: 100% 756/756 [00:00<00:00, 5.56MB/s]\n",
            "(…)-00000-of-00001-dce3832cbf0b04e0.parquet: 100% 2.32M/2.32M [00:00<00:00, 15.7MB/s]\n",
            "Generating train split: 100% 1000/1000 [00:00<00:00, 14979.23 examples/s]\n",
            "[2025-01-02 12:58:41,417] [INFO] [axolotl.get_dataset_wrapper:613] [PID:2106] [RANK:0] Loading dataset with base_type: alpaca and prompt_style: None\u001b[39m\n",
            "Tokenizing Prompts (num_proc=2): 100% 1000/1000 [00:09<00:00, 106.67 examples/s]\n",
            "[2025-01-02 12:58:50,920] [DEBUG] [axolotl.process_datasets_for_packing:189] [PID:2106] [RANK:0] min_input_len: 1277\u001b[39m\n",
            "[2025-01-02 12:58:50,922] [DEBUG] [axolotl.process_datasets_for_packing:191] [PID:2106] [RANK:0] max_input_len: 2083\u001b[39m\n",
            "Dropping Long Sequences (num_proc=2): 100% 1000/1000 [00:02<00:00, 494.63 examples/s]\n",
            "\u001b[33m[2025-01-02 12:58:53,038] [WARNING] [axolotl.process_datasets_for_packing:215] [PID:2106] [RANK:0] Dropped 13 long samples from train dataset\u001b[39m\n",
            "Drop Samples with Zero Trainable Tokens (num_proc=2): 100% 987/987 [00:02<00:00, 450.01 examples/s]\n",
            "Add position_id column (Sample Packing) (num_proc=2): 100% 987/987 [00:01<00:00, 853.46 examples/s] \n",
            "[2025-01-02 12:58:56,551] [INFO] [axolotl.load_tokenized_prepared_datasets:486] [PID:2106] [RANK:0] Saving merged prepared dataset to disk... last_run_prepared/d4980cf45b7d886e0a2393e374eb74e7\u001b[39m\n",
            "Saving the dataset (1/1 shards): 100% 987/987 [00:00<00:00, 19581.10 examples/s]\n",
            "[2025-01-02 12:58:56,618] [DEBUG] [axolotl.calculate_total_num_steps:342] [PID:2106] [RANK:0] total_num_tokens: 1_475_271\u001b[39m\n",
            "[2025-01-02 12:58:56,632] [DEBUG] [axolotl.calculate_total_num_steps:360] [PID:2106] [RANK:0] `total_supervised_tokens: 1_042_722`\u001b[39m\n",
            "[2025-01-02 12:59:03,687] [INFO] [axolotl.utils.samplers.multipack.calc_min_len:197] [PID:2106] [RANK:0] gather_len_batches: [484]\u001b[39m\n",
            "[2025-01-02 12:59:03,687] [DEBUG] [axolotl.calculate_total_num_steps:412] [PID:2106] [RANK:0] data_loader_len: 241\u001b[39m\n",
            "[2025-01-02 12:59:03,688] [INFO] [axolotl.calc_sample_packing_eff_est:418] [PID:2106] [RANK:0] sample_packing_eff_est across ranks: [0.7449298531217684]\u001b[39m\n",
            "[2025-01-02 12:59:03,688] [DEBUG] [axolotl.calculate_total_num_steps:430] [PID:2106] [RANK:0] sample_packing_eff_est: 0.75\u001b[39m\n",
            "[2025-01-02 12:59:03,688] [DEBUG] [axolotl.calculate_total_num_steps:438] [PID:2106] [RANK:0] total_num_steps: 723\u001b[39m\n",
            "[2025-01-02 12:59:03,698] [DEBUG] [axolotl.train.train:66] [PID:2106] [RANK:0] loading tokenizer... TinyLlama/TinyLlama_v1.1\u001b[39m\n",
            "[2025-01-02 12:59:04,210] [DEBUG] [axolotl.load_tokenizer:296] [PID:2106] [RANK:0] EOS: 2 / </s>\u001b[39m\n",
            "[2025-01-02 12:59:04,210] [DEBUG] [axolotl.load_tokenizer:297] [PID:2106] [RANK:0] BOS: 1 / <s>\u001b[39m\n",
            "[2025-01-02 12:59:04,210] [DEBUG] [axolotl.load_tokenizer:298] [PID:2106] [RANK:0] PAD: 2 / </s>\u001b[39m\n",
            "[2025-01-02 12:59:04,210] [DEBUG] [axolotl.load_tokenizer:299] [PID:2106] [RANK:0] UNK: 0 / <unk>\u001b[39m\n",
            "[2025-01-02 12:59:04,210] [INFO] [axolotl.load_tokenizer:313] [PID:2106] [RANK:0] No Chat template selected. Consider adding a chat template for easier inference.\u001b[39m\n",
            "[2025-01-02 12:59:04,210] [DEBUG] [axolotl.train.train:98] [PID:2106] [RANK:0] loading model and peft_config...\u001b[39m\n",
            "[2025-01-02 12:59:04,350] [INFO] [axolotl.monkeypatch.trainer_grad_accum.patch_forward_for_ga:203] [PID:2106] [RANK:0] patching forward\u001b[39m\n",
            "[2025-01-02 12:59:04,355] [INFO] [axolotl.patch_llama_derived_model:560] [PID:2106] [RANK:0] patching llama _prepare_4d_causal_attention_mask*\u001b[39m\n",
            "`low_cpu_mem_usage` was None, now default to True since model is quantized.\n",
            "pytorch_model.bin: 100% 4.40G/4.40G [01:44<00:00, 42.0MB/s]\n",
            "generation_config.json: 100% 129/129 [00:00<00:00, 890kB/s]\n",
            "[2025-01-02 13:01:05,302] [INFO] [axolotl.load_model:1088] [PID:2106] [RANK:0] cuda memory usage after model load: 0.719GB (+0.043GB cache, +0.368GB misc)\u001b[39m\n",
            "[2025-01-02 13:01:05,334] [INFO] [axolotl.prepare_model:1008] [PID:2106] [RANK:0] converting PEFT model w/ prepare_model_for_kbit_training\u001b[39m\n",
            "[2025-01-02 13:01:05,337] [INFO] [axolotl.load_model:1121] [PID:2106] [RANK:0] Converting modules to torch.bfloat16\u001b[39m\n",
            "[2025-01-02 13:01:05,340] [INFO] [axolotl.load_lora:1310] [PID:2106] [RANK:0] found linear modules: ['down_proj', 'gate_proj', 'k_proj', 'o_proj', 'q_proj', 'up_proj', 'v_proj']\u001b[39m\n",
            "trainable params: 25,231,360 || all params: 1,125,279,744 || trainable%: 2.2422\n",
            "[2025-01-02 13:01:05,792] [INFO] [axolotl.load_model:1182] [PID:2106] [RANK:0] cuda memory usage after adapters: 0.813GB (+0.535GB cache, +0.368GB misc)\u001b[39m\n",
            "/usr/local/lib/python3.10/dist-packages/axolotl/core/trainer_builder.py:444: FutureWarning: `tokenizer` is deprecated and will be removed in version 5.0.0 for `AxolotlTrainer.__init__`. Use `processing_class` instead.\n",
            "  super().__init__(*_args, **kwargs)\n",
            "[2025-01-02 13:01:07,230] [INFO] [axolotl.train.train:141] [PID:2106] [RANK:0] Pre-saving adapter config to ./qlora-out\u001b[39m\n",
            "[2025-01-02 13:01:07,234] [INFO] [axolotl.train.train:178] [PID:2106] [RANK:0] Starting trainer...\u001b[39m\n",
            "[2025-01-02 13:01:07,731] [INFO] [axolotl.utils.samplers.multipack.calc_min_len:197] [PID:2106] [RANK:0] gather_len_batches: [484]\u001b[39m\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Currently logged in as: \u001b[33mmohanad\u001b[0m (\u001b[33mmohanad-king-saud-university\u001b[0m). Use \u001b[1m`wandb login --relogin`\u001b[0m to force relogin\n",
            "/usr/local/lib/python3.10/dist-packages/pydantic/main.py:314: UserWarning: Pydantic serializer warnings:\n",
            "  Expected `list[str]` but got `tuple` - serialized value may not be as expected\n",
            "  return self.__pydantic_serializer__.to_python(\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Tracking run with wandb version 0.19.1\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Run data is saved locally in \u001b[35m\u001b[1m/content/wandb/run-20250102_130108-j25saszt\u001b[0m\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Run \u001b[1m`wandb offline`\u001b[0m to turn off syncing.\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Syncing run \u001b[33mefficient-aardvark-6\u001b[0m\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: ⭐️ View project at \u001b[34m\u001b[4mhttps://wandb.ai/mohanad-king-saud-university/axolotl\u001b[0m\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: 🚀 View run at \u001b[34m\u001b[4mhttps://wandb.ai/mohanad-king-saud-university/axolotl/runs/j25saszt\u001b[0m\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[33mWARNING\u001b[0m Saving files without folders. If you want to preserve subdirectories pass base_path to wandb.save, i.e. wandb.save(\"/mnt/folder/file.h5\", base_path=\"/mnt\")\n",
            "[2025-01-02 13:01:10,200] [INFO] [axolotl.callbacks.on_train_begin:814] [PID:2106] [RANK:0] The Axolotl config has been saved to the WandB run under files.\u001b[39m\n",
            "{'loss': 0.7502, 'grad_norm': 0.15253041684627533, 'learning_rate': 2e-05, 'epoch': 0.0}\n",
            "  0% 1/723 [00:28<5:44:30, 28.63s/it][2025-01-02 13:02:01,680] [INFO] [axolotl.callbacks.on_step_end:130] [PID:2106] [RANK:0] cuda memory usage while training: 0.928GB (+7.949GB cache, +0.636GB misc)\u001b[39m\n",
            "{'loss': 0.7483, 'grad_norm': 0.17908836901187897, 'learning_rate': 4e-05, 'epoch': 0.01}\n",
            "{'loss': 0.5611, 'grad_norm': 0.1436198353767395, 'learning_rate': 6e-05, 'epoch': 0.01}\n",
            "{'loss': 0.464, 'grad_norm': 0.10988904535770416, 'learning_rate': 8e-05, 'epoch': 0.02}\n",
            "{'loss': 0.5042, 'grad_norm': 0.12218518555164337, 'learning_rate': 0.0001, 'epoch': 0.02}\n",
            "{'loss': 0.541, 'grad_norm': 0.12142739444971085, 'learning_rate': 0.00012, 'epoch': 0.02}\n",
            "{'loss': 0.6585, 'grad_norm': 0.1377066969871521, 'learning_rate': 0.00014, 'epoch': 0.03}\n",
            "{'loss': 0.6652, 'grad_norm': 0.12895096838474274, 'learning_rate': 0.00016, 'epoch': 0.03}\n",
            "  1% 8/723 [03:36<5:29:17, 27.63s/it]\n",
            "  0% 0/10 [00:00<?, ?it/s]\u001b[A\n",
            " 20% 2/10 [00:03<00:12,  1.60s/it]\u001b[A\n",
            " 30% 3/10 [00:06<00:16,  2.29s/it]\u001b[A\n",
            " 40% 4/10 [00:09<00:15,  2.63s/it]\u001b[A\n",
            " 50% 5/10 [00:12<00:13,  2.79s/it]\u001b[A\n",
            " 60% 6/10 [00:15<00:11,  2.86s/it]\u001b[A\n",
            " 70% 7/10 [00:18<00:08,  2.91s/it]\u001b[A\n",
            " 80% 8/10 [00:21<00:05,  2.94s/it]\u001b[A\n",
            " 90% 9/10 [00:24<00:02,  2.97s/it]\u001b[A[2025-01-02 13:05:16,105] [INFO] [accelerate.accelerator.gather_for_metrics:2509] [PID:2106] The used dataset had no length, returning gathered tensors. You should drop the remainder yourself.\n",
            "\n",
            "                                     \n",
            "\u001b[A{'eval_loss': 0.7287575006484985, 'eval_runtime': 30.9807, 'eval_samples_per_second': 0.646, 'eval_steps_per_second': 0.323, 'epoch': 0.03}\n",
            "  1% 8/723 [04:07<5:29:17, 27.63s/it]\n",
            "100% 10/10 [00:29<00:00,  2.98s/it]\u001b[A\n",
            "{'loss': 0.5634, 'grad_norm': 0.1398961991071701, 'learning_rate': 0.00018, 'epoch': 0.04}\n",
            "{'loss': 0.69, 'grad_norm': 0.15945462882518768, 'learning_rate': 0.0002, 'epoch': 0.04}\n",
            "{'loss': 0.7582, 'grad_norm': 0.21529807150363922, 'learning_rate': 0.00019999902928891875, 'epoch': 0.05}\n",
            "{'loss': 0.4648, 'grad_norm': 0.13121061027050018, 'learning_rate': 0.00019999611717452052, 'epoch': 0.05}\n",
            "{'loss': 0.5722, 'grad_norm': 0.1718645989894867, 'learning_rate': 0.00019999126371334178, 'epoch': 0.05}\n",
            "{'loss': 0.5103, 'grad_norm': 0.1729479432106018, 'learning_rate': 0.0001999844689996087, 'epoch': 0.06}\n",
            "{'loss': 0.7867, 'grad_norm': 0.2179705798625946, 'learning_rate': 0.0001999757331652354, 'epoch': 0.06}\n",
            "{'loss': 0.6661, 'grad_norm': 0.20792384445667267, 'learning_rate': 0.00019996505637982122, 'epoch': 0.07}\n",
            "  2% 16/723 [07:41<5:22:01, 27.33s/it]\n",
            "  0% 0/10 [00:00<?, ?it/s]\u001b[A\n",
            " 20% 2/10 [00:03<00:12,  1.60s/it]\u001b[A\n",
            " 30% 3/10 [00:06<00:16,  2.30s/it]\u001b[A\n",
            " 40% 4/10 [00:09<00:15,  2.63s/it]\u001b[A\n",
            " 50% 5/10 [00:12<00:14,  2.80s/it]\u001b[A\n",
            " 60% 6/10 [00:15<00:11,  2.87s/it]\u001b[A\n",
            " 70% 7/10 [00:18<00:08,  2.92s/it]\u001b[A\n",
            " 80% 8/10 [00:21<00:05,  2.95s/it]\u001b[A\n",
            " 90% 9/10 [00:24<00:02,  2.97s/it]\u001b[A[2025-01-02 13:09:21,076] [INFO] [accelerate.accelerator.gather_for_metrics:2509] [PID:2106] The used dataset had no length, returning gathered tensors. You should drop the remainder yourself.\n",
            "\n",
            "                                      \n",
            "\u001b[A{'eval_loss': 0.6923397183418274, 'eval_runtime': 31.0082, 'eval_samples_per_second': 0.645, 'eval_steps_per_second': 0.322, 'epoch': 0.07}\n",
            "  2% 16/723 [08:12<5:22:01, 27.33s/it]\n",
            "100% 10/10 [00:29<00:00,  2.98s/it]\u001b[A\n",
            "{'loss': 0.6659, 'grad_norm': 0.22102929651737213, 'learning_rate': 0.00019995243885064765, 'epoch': 0.07}\n",
            "{'loss': 0.412, 'grad_norm': 0.15632086992263794, 'learning_rate': 0.00019993788082267429, 'epoch': 0.07}\n",
            "{'loss': 0.5537, 'grad_norm': 0.17257443070411682, 'learning_rate': 0.0001999213825785338, 'epoch': 0.08}\n",
            "{'loss': 0.8784, 'grad_norm': 0.1833968162536621, 'learning_rate': 0.00019990294443852685, 'epoch': 0.08}\n",
            "{'loss': 0.6239, 'grad_norm': 0.1756899654865265, 'learning_rate': 0.00019988256676061554, 'epoch': 0.09}\n",
            "{'loss': 0.5552, 'grad_norm': 0.171953022480011, 'learning_rate': 0.00019986024994041662, 'epoch': 0.09}\n",
            "{'loss': 0.5108, 'grad_norm': 0.16000303626060486, 'learning_rate': 0.00019983599441119373, 'epoch': 0.1}\n",
            "{'loss': 0.5459, 'grad_norm': 0.15385715663433075, 'learning_rate': 0.00019980980064384916, 'epoch': 0.1}\n",
            "  3% 24/723 [11:26<5:02:40, 25.98s/it]\n",
            "  0% 0/10 [00:00<?, ?it/s]\u001b[A\n",
            " 20% 2/10 [00:03<00:12,  1.60s/it]\u001b[A\n",
            " 30% 3/10 [00:06<00:16,  2.30s/it]\u001b[A\n",
            " 40% 4/10 [00:09<00:15,  2.64s/it]\u001b[A\n",
            " 50% 5/10 [00:12<00:13,  2.80s/it]\u001b[A\n",
            " 60% 6/10 [00:15<00:11,  2.87s/it]\u001b[A\n",
            " 70% 7/10 [00:18<00:08,  2.92s/it]\u001b[A\n",
            " 80% 8/10 [00:21<00:05,  2.95s/it]\u001b[A\n",
            " 90% 9/10 [00:24<00:02,  2.97s/it]\u001b[A[2025-01-02 13:13:06,004] [INFO] [accelerate.accelerator.gather_for_metrics:2509] [PID:2106] The used dataset had no length, returning gathered tensors. You should drop the remainder yourself.\n",
            "\n",
            "                                      \n",
            "\u001b[A{'eval_loss': 0.6747424006462097, 'eval_runtime': 30.9785, 'eval_samples_per_second': 0.646, 'eval_steps_per_second': 0.323, 'epoch': 0.1}\n",
            "  3% 24/723 [11:57<5:02:40, 25.98s/it]\n",
            "100% 10/10 [00:29<00:00,  2.98s/it]\u001b[A\n",
            "{'loss': 0.6031, 'grad_norm': 0.1544913649559021, 'learning_rate': 0.00019978166914691453, 'epoch': 0.1}\n",
            "{'loss': 0.4047, 'grad_norm': 0.1293278932571411, 'learning_rate': 0.0001997516004665409, 'epoch': 0.11}\n",
            "{'loss': 0.5748, 'grad_norm': 0.16374485194683075, 'learning_rate': 0.00019971959518648834, 'epoch': 0.11}\n",
            "{'loss': 0.84, 'grad_norm': 0.20005881786346436, 'learning_rate': 0.0001996856539281144, 'epoch': 0.12}\n",
            "{'loss': 0.54, 'grad_norm': 0.1523049920797348, 'learning_rate': 0.00019964977735036223, 'epoch': 0.12}\n",
            "{'loss': 0.7926, 'grad_norm': 0.1651250272989273, 'learning_rate': 0.00019961196614974767, 'epoch': 0.12}\n",
            "{'loss': 0.7594, 'grad_norm': 0.16529132425785065, 'learning_rate': 0.00019957222106034572, 'epoch': 0.13}\n",
            "{'loss': 0.4096, 'grad_norm': 0.1586233675479889, 'learning_rate': 0.00019953054285377634, 'epoch': 0.13}\n",
            "  4% 32/723 [15:12<4:41:16, 24.42s/it]\n",
            "  0% 0/10 [00:00<?, ?it/s]\u001b[A\n",
            " 20% 2/10 [00:03<00:12,  1.60s/it]\u001b[A\n",
            " 30% 3/10 [00:06<00:16,  2.30s/it]\u001b[A\n",
            " 40% 4/10 [00:09<00:15,  2.64s/it]\u001b[A\n",
            " 50% 5/10 [00:12<00:14,  2.80s/it]\u001b[A\n",
            " 60% 6/10 [00:15<00:11,  2.87s/it]\u001b[A\n",
            " 70% 7/10 [00:18<00:08,  2.92s/it]\u001b[A\n",
            " 80% 8/10 [00:21<00:05,  2.95s/it]\u001b[A\n",
            " 90% 9/10 [00:24<00:02,  2.97s/it]\u001b[A[2025-01-02 13:16:52,613] [INFO] [accelerate.accelerator.gather_for_metrics:2509] [PID:2106] The used dataset had no length, returning gathered tensors. You should drop the remainder yourself.\n",
            "\n",
            "                                      \n",
            "\u001b[A{'eval_loss': 0.6655129194259644, 'eval_runtime': 31.0173, 'eval_samples_per_second': 0.645, 'eval_steps_per_second': 0.322, 'epoch': 0.13}\n",
            "  4% 32/723 [15:43<4:41:16, 24.42s/it]\n",
            "100% 10/10 [00:29<00:00,  2.99s/it]\u001b[A\n",
            "{'loss': 0.5213, 'grad_norm': 0.14884868264198303, 'learning_rate': 0.00019948693233918952, 'epoch': 0.14}\n",
            "{'loss': 0.5055, 'grad_norm': 0.15160582959651947, 'learning_rate': 0.00019944139036324942, 'epoch': 0.14}\n",
            "{'loss': 0.6103, 'grad_norm': 0.1714474856853485, 'learning_rate': 0.00019939391781011807, 'epoch': 0.14}\n",
            "{'loss': 0.6213, 'grad_norm': 0.1517469435930252, 'learning_rate': 0.00019934451560143815, 'epoch': 0.15}\n",
            "{'loss': 0.595, 'grad_norm': 0.1401410698890686, 'learning_rate': 0.00019929318469631504, 'epoch': 0.15}\n",
            "{'loss': 0.6023, 'grad_norm': 0.18747936189174652, 'learning_rate': 0.0001992399260912983, 'epoch': 0.16}\n",
            "{'loss': 0.5723, 'grad_norm': 0.16074074804782867, 'learning_rate': 0.0001991847408203624, 'epoch': 0.16}\n",
            "{'loss': 0.4572, 'grad_norm': 0.16260217130184174, 'learning_rate': 0.00019912762995488633, 'epoch': 0.17}\n",
            "  6% 40/723 [19:11<4:55:15, 25.94s/it]\n",
            "  0% 0/10 [00:00<?, ?it/s]\u001b[A\n",
            " 20% 2/10 [00:03<00:12,  1.60s/it]\u001b[A\n",
            " 30% 3/10 [00:06<00:16,  2.31s/it]\u001b[A\n",
            " 40% 4/10 [00:09<00:15,  2.64s/it]\u001b[A\n",
            " 50% 5/10 [00:12<00:14,  2.80s/it]\u001b[A\n",
            " 60% 6/10 [00:15<00:11,  2.87s/it]\u001b[A\n",
            " 70% 7/10 [00:18<00:08,  2.92s/it]\u001b[A\n",
            " 80% 8/10 [00:21<00:05,  2.95s/it]\u001b[A\n",
            " 90% 9/10 [00:24<00:02,  2.97s/it]\u001b[A[2025-01-02 13:20:51,445] [INFO] [accelerate.accelerator.gather_for_metrics:2509] [PID:2106] The used dataset had no length, returning gathered tensors. You should drop the remainder yourself.\n",
            "\n",
            "                                      \n",
            "\u001b[A{'eval_loss': 0.6578646898269653, 'eval_runtime': 31.0247, 'eval_samples_per_second': 0.645, 'eval_steps_per_second': 0.322, 'epoch': 0.17}\n",
            "  6% 40/723 [19:42<4:55:15, 25.94s/it]\n",
            "100% 10/10 [00:29<00:00,  2.98s/it]\u001b[A\n",
            "{'loss': 0.6983, 'grad_norm': 0.18567661941051483, 'learning_rate': 0.00019906859460363307, 'epoch': 0.17}\n",
            "{'loss': 0.7265, 'grad_norm': 0.17334288358688354, 'learning_rate': 0.00019900763591272812, 'epoch': 0.17}\n",
            "{'loss': 0.4641, 'grad_norm': 0.15245021879673004, 'learning_rate': 0.00019894475506563688, 'epoch': 0.18}\n",
            "{'loss': 0.5257, 'grad_norm': 0.19029416143894196, 'learning_rate': 0.00019887995328314215, 'epoch': 0.18}\n",
            "{'loss': 0.6475, 'grad_norm': 0.1779717057943344, 'learning_rate': 0.00019881323182332006, 'epoch': 0.19}\n",
            "{'loss': 0.4647, 'grad_norm': 0.16865681111812592, 'learning_rate': 0.00019874459198151583, 'epoch': 0.19}\n",
            "{'loss': 0.5318, 'grad_norm': 0.172408789396286, 'learning_rate': 0.00019867403509031854, 'epoch': 0.19}\n",
            "{'loss': 0.408, 'grad_norm': 0.19622324407100677, 'learning_rate': 0.00019860156251953534, 'epoch': 0.2}\n",
            "  7% 48/723 [23:01<4:56:42, 26.37s/it]\n",
            "  0% 0/10 [00:00<?, ?it/s]\u001b[A\n",
            " 20% 2/10 [00:03<00:12,  1.60s/it]\u001b[A\n",
            " 30% 3/10 [00:06<00:16,  2.30s/it]\u001b[A\n",
            " 40% 4/10 [00:09<00:15,  2.63s/it]\u001b[A\n",
            " 50% 5/10 [00:12<00:13,  2.80s/it]\u001b[A\n",
            " 60% 6/10 [00:15<00:11,  2.87s/it]\u001b[A\n",
            " 70% 7/10 [00:18<00:08,  2.91s/it]\u001b[A\n",
            " 80% 8/10 [00:21<00:05,  2.95s/it]\u001b[A\n",
            " 90% 9/10 [00:24<00:02,  2.97s/it]\u001b[A[2025-01-02 13:24:41,958] [INFO] [accelerate.accelerator.gather_for_metrics:2509] [PID:2106] The used dataset had no length, returning gathered tensors. You should drop the remainder yourself.\n",
            "\n",
            "                                      \n",
            "\u001b[A{'eval_loss': 0.6532100439071655, 'eval_runtime': 30.9668, 'eval_samples_per_second': 0.646, 'eval_steps_per_second': 0.323, 'epoch': 0.2}\n",
            "  7% 48/723 [23:32<4:56:42, 26.37s/it]\n",
            "100% 10/10 [00:29<00:00,  2.98s/it]\u001b[A\n",
            "{'loss': 0.5151, 'grad_norm': 0.1742960661649704, 'learning_rate': 0.00019852717567616477, 'epoch': 0.2}\n",
            "{'loss': 0.4692, 'grad_norm': 0.1681739091873169, 'learning_rate': 0.00019845087600436947, 'epoch': 0.21}\n",
            "{'loss': 0.4884, 'grad_norm': 0.13979633152484894, 'learning_rate': 0.0001983726649854482, 'epoch': 0.21}\n",
            "{'loss': 0.5462, 'grad_norm': 0.13928000628948212, 'learning_rate': 0.00019829254413780706, 'epoch': 0.22}\n",
            "{'loss': 0.5141, 'grad_norm': 0.16459038853645325, 'learning_rate': 0.00019821051501692986, 'epoch': 0.22}\n",
            "{'loss': 0.8518, 'grad_norm': 0.16649477183818817, 'learning_rate': 0.00019812657921534818, 'epoch': 0.22}\n",
            "{'loss': 0.5383, 'grad_norm': 0.1618327647447586, 'learning_rate': 0.00019804073836261025, 'epoch': 0.23}\n",
            "{'loss': 0.5586, 'grad_norm': 0.18050715327262878, 'learning_rate': 0.00019795299412524945, 'epoch': 0.23}\n",
            "  8% 56/723 [26:54<4:44:34, 25.60s/it]\n",
            "  0% 0/10 [00:00<?, ?it/s]\u001b[A\n",
            " 20% 2/10 [00:03<00:12,  1.60s/it]\u001b[A\n",
            " 30% 3/10 [00:06<00:16,  2.30s/it]\u001b[A\n",
            " 40% 4/10 [00:09<00:15,  2.63s/it]\u001b[A\n",
            " 50% 5/10 [00:12<00:13,  2.79s/it]\u001b[A\n",
            " 60% 6/10 [00:15<00:11,  2.87s/it]\u001b[A\n",
            " 70% 7/10 [00:18<00:08,  2.91s/it]\u001b[A\n",
            " 80% 8/10 [00:21<00:05,  2.94s/it]\u001b[A\n",
            " 90% 9/10 [00:24<00:02,  2.96s/it]\u001b[A[2025-01-02 13:28:34,472] [INFO] [accelerate.accelerator.gather_for_metrics:2509] [PID:2106] The used dataset had no length, returning gathered tensors. You should drop the remainder yourself.\n",
            "\n",
            "                                      \n",
            "\u001b[A{'eval_loss': 0.6500886678695679, 'eval_runtime': 30.9542, 'eval_samples_per_second': 0.646, 'eval_steps_per_second': 0.323, 'epoch': 0.23}\n",
            "  8% 56/723 [27:25<4:44:34, 25.60s/it]\n",
            "100% 10/10 [00:29<00:00,  2.98s/it]\u001b[A\n",
            "{'loss': 0.4633, 'grad_norm': 0.14873482286930084, 'learning_rate': 0.00019786334820675184, 'epoch': 0.24}\n",
            "{'loss': 0.4769, 'grad_norm': 0.16331404447555542, 'learning_rate': 0.00019777180234752307, 'epoch': 0.24}\n",
            "{'loss': 0.6718, 'grad_norm': 0.21056173741817474, 'learning_rate': 0.00019767835832485485, 'epoch': 0.24}\n",
            "{'loss': 0.6943, 'grad_norm': 0.16289901733398438, 'learning_rate': 0.0001975830179528901, 'epoch': 0.25}\n",
            "{'loss': 0.5024, 'grad_norm': 0.14962373673915863, 'learning_rate': 0.0001974857830825879, 'epoch': 0.25}\n",
            "{'loss': 0.6386, 'grad_norm': 0.16236631572246552, 'learning_rate': 0.00019738665560168763, 'epoch': 0.26}\n",
            "{'loss': 0.6637, 'grad_norm': 0.15196746587753296, 'learning_rate': 0.00019728563743467214, 'epoch': 0.26}\n",
            "{'loss': 0.6441, 'grad_norm': 0.16023871302604675, 'learning_rate': 0.00019718273054273051, 'epoch': 0.27}\n",
            "  9% 64/723 [31:06<5:01:09, 27.42s/it]\n",
            "  0% 0/10 [00:00<?, ?it/s]\u001b[A\n",
            " 20% 2/10 [00:03<00:12,  1.60s/it]\u001b[A\n",
            " 30% 3/10 [00:06<00:16,  2.30s/it]\u001b[A\n",
            " 40% 4/10 [00:09<00:15,  2.64s/it]\u001b[A\n",
            " 50% 5/10 [00:12<00:14,  2.80s/it]\u001b[A\n",
            " 60% 6/10 [00:15<00:11,  2.87s/it]\u001b[A\n",
            " 70% 7/10 [00:18<00:08,  2.92s/it]\u001b[A\n",
            " 80% 8/10 [00:21<00:05,  2.95s/it]\u001b[A\n",
            " 90% 9/10 [00:24<00:02,  2.97s/it]\u001b[A[2025-01-02 13:32:46,329] [INFO] [accelerate.accelerator.gather_for_metrics:2509] [PID:2106] The used dataset had no length, returning gathered tensors. You should drop the remainder yourself.\n",
            "\n",
            "                                      \n",
            "\u001b[A{'eval_loss': 0.6454617977142334, 'eval_runtime': 31.0096, 'eval_samples_per_second': 0.645, 'eval_steps_per_second': 0.322, 'epoch': 0.27}\n",
            "  9% 64/723 [31:37<5:01:09, 27.42s/it]\n",
            "100% 10/10 [00:29<00:00,  2.98s/it]\u001b[A\n",
            "{'loss': 0.5654, 'grad_norm': 0.1434909552335739, 'learning_rate': 0.00019707793692372, 'epoch': 0.27}\n",
            "{'loss': 0.5142, 'grad_norm': 0.16578759253025055, 'learning_rate': 0.00019697125861212707, 'epoch': 0.27}\n",
            "{'loss': 0.4664, 'grad_norm': 0.1614455282688141, 'learning_rate': 0.00019686269767902812, 'epoch': 0.28}\n",
            "{'loss': 0.2861, 'grad_norm': 0.11947987973690033, 'learning_rate': 0.0001967522562320492, 'epoch': 0.28}\n",
            "{'loss': 0.657, 'grad_norm': 0.1741272509098053, 'learning_rate': 0.00019663993641532508, 'epoch': 0.29}\n",
            "{'loss': 0.4787, 'grad_norm': 0.15194806456565857, 'learning_rate': 0.00019652574040945745, 'epoch': 0.29}\n",
            "{'loss': 0.6643, 'grad_norm': 0.17933715879917145, 'learning_rate': 0.00019640967043147302, 'epoch': 0.29}\n",
            "{'loss': 0.3884, 'grad_norm': 0.14513811469078064, 'learning_rate': 0.00019629172873477995, 'epoch': 0.3}\n",
            " 10% 72/723 [34:54<4:23:48, 24.31s/it]\n",
            "  0% 0/10 [00:00<?, ?it/s]\u001b[A\n",
            " 20% 2/10 [00:03<00:12,  1.60s/it]\u001b[A\n",
            " 30% 3/10 [00:06<00:16,  2.30s/it]\u001b[A\n",
            " 40% 4/10 [00:09<00:15,  2.63s/it]\u001b[A\n",
            " 50% 5/10 [00:12<00:13,  2.80s/it]\u001b[A\n",
            " 60% 6/10 [00:15<00:11,  2.87s/it]\u001b[A\n",
            " 70% 7/10 [00:18<00:08,  2.92s/it]\u001b[A\n",
            " 80% 8/10 [00:21<00:05,  2.95s/it]\u001b[A\n",
            " 90% 9/10 [00:24<00:02,  2.97s/it]\u001b[A[2025-01-02 13:36:34,427] [INFO] [accelerate.accelerator.gather_for_metrics:2509] [PID:2106] The used dataset had no length, returning gathered tensors. You should drop the remainder yourself.\n",
            "\n",
            "                                      \n",
            "\u001b[A{'eval_loss': 0.6429021954536438, 'eval_runtime': 30.9933, 'eval_samples_per_second': 0.645, 'eval_steps_per_second': 0.323, 'epoch': 0.3}\n",
            " 10% 72/723 [35:25<4:23:48, 24.31s/it]\n",
            "100% 10/10 [00:29<00:00,  2.98s/it]\u001b[A\n",
            "{'loss': 0.625, 'grad_norm': 0.1758933961391449, 'learning_rate': 0.0001961719176091245, 'epoch': 0.3}\n",
            "{'loss': 0.4108, 'grad_norm': 0.14782272279262543, 'learning_rate': 0.0001960502393805465, 'epoch': 0.31}\n",
            "{'loss': 0.5491, 'grad_norm': 0.1533605009317398, 'learning_rate': 0.00019592669641133395, 'epoch': 0.31}\n",
            "{'loss': 0.5998, 'grad_norm': 0.19208893179893494, 'learning_rate': 0.0001958012910999775, 'epoch': 0.31}\n",
            "{'loss': 0.6879, 'grad_norm': 0.19715486466884613, 'learning_rate': 0.0001956740258811236, 'epoch': 0.32}\n",
            "{'loss': 0.3907, 'grad_norm': 0.13977134227752686, 'learning_rate': 0.00019554490322552744, 'epoch': 0.32}\n",
            "{'loss': 0.5149, 'grad_norm': 0.2247178703546524, 'learning_rate': 0.00019541392564000488, 'epoch': 0.33}\n",
            "{'loss': 0.4677, 'grad_norm': 0.1537841558456421, 'learning_rate': 0.00019528109566738382, 'epoch': 0.33}\n",
            " 11% 80/723 [38:43<4:37:02, 25.85s/it]\n",
            "  0% 0/10 [00:00<?, ?it/s]\u001b[A\n",
            " 20% 2/10 [00:03<00:12,  1.60s/it]\u001b[A\n",
            " 30% 3/10 [00:06<00:16,  2.31s/it]\u001b[A\n",
            " 40% 4/10 [00:09<00:15,  2.64s/it]\u001b[A\n",
            " 50% 5/10 [00:12<00:13,  2.80s/it]\u001b[A\n",
            " 60% 6/10 [00:15<00:11,  2.87s/it]\u001b[A\n",
            " 70% 7/10 [00:18<00:08,  2.92s/it]\u001b[A\n",
            " 80% 8/10 [00:21<00:05,  2.95s/it]\u001b[A\n",
            " 90% 9/10 [00:24<00:02,  2.97s/it]\u001b[A[2025-01-02 13:40:23,846] [INFO] [accelerate.accelerator.gather_for_metrics:2509] [PID:2106] The used dataset had no length, returning gathered tensors. You should drop the remainder yourself.\n",
            "\n",
            "                                      \n",
            "\u001b[A{'eval_loss': 0.6401460766792297, 'eval_runtime': 30.9897, 'eval_samples_per_second': 0.645, 'eval_steps_per_second': 0.323, 'epoch': 0.33}\n",
            " 11% 80/723 [39:14<4:37:02, 25.85s/it]\n",
            "100% 10/10 [00:29<00:00,  2.98s/it]\u001b[A\n",
            "{'loss': 0.4969, 'grad_norm': 0.17191514372825623, 'learning_rate': 0.00019514641588645471, 'epoch': 0.34}\n",
            "{'loss': 0.7995, 'grad_norm': 0.16509486734867096, 'learning_rate': 0.00019500988891192074, 'epoch': 0.34}\n",
            "{'loss': 0.5211, 'grad_norm': 0.13457974791526794, 'learning_rate': 0.00019487151739434685, 'epoch': 0.34}\n",
            "{'loss': 0.4891, 'grad_norm': 0.16758953034877777, 'learning_rate': 0.00019473130402010829, 'epoch': 0.35}\n",
            "{'loss': 0.5812, 'grad_norm': 0.196170374751091, 'learning_rate': 0.0001945892515113386, 'epoch': 0.35}\n",
            "{'loss': 0.652, 'grad_norm': 0.18328820168972015, 'learning_rate': 0.00019444536262587669, 'epoch': 0.36}\n",
            "{'loss': 0.5377, 'grad_norm': 0.18717023730278015, 'learning_rate': 0.00019429964015721328, 'epoch': 0.36}\n",
            "{'loss': 0.4715, 'grad_norm': 0.14570501446723938, 'learning_rate': 0.00019415208693443665, 'epoch': 0.36}\n",
            " 12% 88/723 [42:38<4:44:46, 26.91s/it]\n",
            "  0% 0/10 [00:00<?, ?it/s]\u001b[A\n",
            " 20% 2/10 [00:03<00:12,  1.60s/it]\u001b[A\n",
            " 30% 3/10 [00:06<00:16,  2.30s/it]\u001b[A\n",
            " 40% 4/10 [00:09<00:15,  2.64s/it]\u001b[A\n",
            " 50% 5/10 [00:12<00:13,  2.80s/it]\u001b[A\n",
            " 60% 6/10 [00:15<00:11,  2.87s/it]\u001b[A\n",
            " 70% 7/10 [00:18<00:08,  2.91s/it]\u001b[A\n",
            " 80% 8/10 [00:21<00:05,  2.94s/it]\u001b[A\n",
            " 90% 9/10 [00:24<00:02,  2.97s/it]\u001b[A[2025-01-02 13:44:18,695] [INFO] [accelerate.accelerator.gather_for_metrics:2509] [PID:2106] The used dataset had no length, returning gathered tensors. You should drop the remainder yourself.\n",
            "\n",
            "                                      \n",
            "\u001b[A{'eval_loss': 0.6376045942306519, 'eval_runtime': 30.978, 'eval_samples_per_second': 0.646, 'eval_steps_per_second': 0.323, 'epoch': 0.36}\n",
            " 12% 88/723 [43:09<4:44:46, 26.91s/it]\n",
            "100% 10/10 [00:29<00:00,  2.98s/it]\u001b[A\n",
            "{'loss': 0.5719, 'grad_norm': 0.17980214953422546, 'learning_rate': 0.00019400270582217775, 'epoch': 0.37}\n",
            "{'loss': 0.6654, 'grad_norm': 0.17548443377017975, 'learning_rate': 0.00019385149972055466, 'epoch': 0.37}\n",
            "{'loss': 0.4929, 'grad_norm': 0.14056533575057983, 'learning_rate': 0.0001936984715651161, 'epoch': 0.38}\n",
            "{'loss': 0.6443, 'grad_norm': 0.16604070365428925, 'learning_rate': 0.00019354362432678462, 'epoch': 0.38}\n",
            "{'loss': 0.454, 'grad_norm': 0.15036888420581818, 'learning_rate': 0.0001933869610117988, 'epoch': 0.39}\n",
            "{'loss': 0.4552, 'grad_norm': 0.14737460017204285, 'learning_rate': 0.00019322848466165495, 'epoch': 0.39}\n",
            "{'loss': 0.5311, 'grad_norm': 0.17734794318675995, 'learning_rate': 0.0001930681983530481, 'epoch': 0.39}\n",
            "{'loss': 0.4688, 'grad_norm': 0.14790746569633484, 'learning_rate': 0.00019290610519781212, 'epoch': 0.4}\n",
            " 13% 96/723 [46:31<4:30:51, 25.92s/it]\n",
            "  0% 0/10 [00:00<?, ?it/s]\u001b[A\n",
            " 20% 2/10 [00:03<00:12,  1.60s/it]\u001b[A\n",
            " 30% 3/10 [00:06<00:16,  2.30s/it]\u001b[A\n",
            " 40% 4/10 [00:09<00:15,  2.64s/it]\u001b[A\n",
            " 50% 5/10 [00:12<00:14,  2.80s/it]\u001b[A\n",
            " 60% 6/10 [00:15<00:11,  2.87s/it]\u001b[A\n",
            " 70% 7/10 [00:18<00:08,  2.92s/it]\u001b[A\n",
            " 80% 8/10 [00:21<00:05,  2.95s/it]\u001b[A\n",
            " 90% 9/10 [00:24<00:02,  2.97s/it]\u001b[A[2025-01-02 13:48:11,040] [INFO] [accelerate.accelerator.gather_for_metrics:2509] [PID:2106] The used dataset had no length, returning gathered tensors. You should drop the remainder yourself.\n",
            "\n",
            "                                      \n",
            "\u001b[A{'eval_loss': 0.6360621452331543, 'eval_runtime': 31.0044, 'eval_samples_per_second': 0.645, 'eval_steps_per_second': 0.323, 'epoch': 0.4}\n",
            " 13% 96/723 [47:02<4:30:51, 25.92s/it]\n",
            "100% 10/10 [00:29<00:00,  2.98s/it]\u001b[A\n",
            "{'loss': 0.462, 'grad_norm': 0.16391420364379883, 'learning_rate': 0.00019274220834285946, 'epoch': 0.4}\n",
            "{'loss': 0.3994, 'grad_norm': 0.15956422686576843, 'learning_rate': 0.00019257651097012005, 'epoch': 0.41}\n",
            "{'loss': 0.5909, 'grad_norm': 0.1684301495552063, 'learning_rate': 0.0001924090162964793, 'epoch': 0.41}\n",
            "{'loss': 0.6875, 'grad_norm': 0.1925661563873291, 'learning_rate': 0.000192239727573716, 'epoch': 0.41}\n",
            "{'loss': 0.626, 'grad_norm': 0.17237542569637299, 'learning_rate': 0.00019206864808843892, 'epoch': 0.42}\n",
            "{'loss': 0.637, 'grad_norm': 0.16767676174640656, 'learning_rate': 0.00019189578116202307, 'epoch': 0.42}\n",
            "{'loss': 0.5876, 'grad_norm': 0.3015708327293396, 'learning_rate': 0.00019172113015054532, 'epoch': 0.43}\n",
            "{'loss': 0.4405, 'grad_norm': 0.13659274578094482, 'learning_rate': 0.00019154469844471906, 'epoch': 0.43}\n",
            " 14% 104/723 [50:24<4:38:59, 27.04s/it]\n",
            "  0% 0/10 [00:00<?, ?it/s]\u001b[A\n",
            " 20% 2/10 [00:03<00:12,  1.60s/it]\u001b[A\n",
            " 30% 3/10 [00:06<00:16,  2.29s/it]\u001b[A\n",
            " 40% 4/10 [00:09<00:15,  2.63s/it]\u001b[A\n",
            " 50% 5/10 [00:12<00:13,  2.79s/it]\u001b[A\n",
            " 60% 6/10 [00:15<00:11,  2.86s/it]\u001b[A\n",
            " 70% 7/10 [00:18<00:08,  2.91s/it]\u001b[A\n",
            " 80% 8/10 [00:21<00:05,  2.94s/it]\u001b[A\n",
            " 90% 9/10 [00:24<00:02,  2.96s/it]\u001b[A[2025-01-02 13:52:04,605] [INFO] [accelerate.accelerator.gather_for_metrics:2509] [PID:2106] The used dataset had no length, returning gathered tensors. You should drop the remainder yourself.\n",
            "\n",
            "                                       \n",
            "\u001b[A{'eval_loss': 0.6343576312065125, 'eval_runtime': 30.9554, 'eval_samples_per_second': 0.646, 'eval_steps_per_second': 0.323, 'epoch': 0.43}\n",
            " 14% 104/723 [50:55<4:38:59, 27.04s/it]\n",
            "100% 10/10 [00:28<00:00,  2.98s/it]\u001b[A\n",
            "{'loss': 0.5545, 'grad_norm': 0.1639268547296524, 'learning_rate': 0.0001913664894698286, 'epoch': 0.43}\n",
            "{'loss': 0.4942, 'grad_norm': 0.14340513944625854, 'learning_rate': 0.0001911865066856624, 'epoch': 0.44}\n",
            "{'loss': 0.5637, 'grad_norm': 0.20544105768203735, 'learning_rate': 0.00019100475358644614, 'epoch': 0.44}\n",
            "{'loss': 0.4945, 'grad_norm': 0.1511404812335968, 'learning_rate': 0.00019082123370077483, 'epoch': 0.45}\n",
            "{'loss': 0.6442, 'grad_norm': 0.16580159962177277, 'learning_rate': 0.0001906359505915441, 'epoch': 0.45}\n",
            "{'loss': 0.6117, 'grad_norm': 0.1789024919271469, 'learning_rate': 0.0001904489078558814, 'epoch': 0.46}\n",
            "{'loss': 0.4467, 'grad_norm': 0.14636443555355072, 'learning_rate': 0.00019026010912507577, 'epoch': 0.46}\n",
            "{'loss': 0.5842, 'grad_norm': 0.18820209801197052, 'learning_rate': 0.00019006955806450765, 'epoch': 0.46}\n",
            " 15% 112/723 [54:14<4:20:24, 25.57s/it]\n",
            "  0% 0/10 [00:00<?, ?it/s]\u001b[A\n",
            " 20% 2/10 [00:03<00:12,  1.60s/it]\u001b[A\n",
            " 30% 3/10 [00:06<00:16,  2.29s/it]\u001b[A\n",
            " 40% 4/10 [00:09<00:15,  2.63s/it]\u001b[A\n",
            " 50% 5/10 [00:12<00:13,  2.80s/it]\u001b[A\n",
            " 60% 6/10 [00:15<00:11,  2.87s/it]\u001b[A\n",
            " 70% 7/10 [00:18<00:08,  2.91s/it]\u001b[A\n",
            " 80% 8/10 [00:21<00:05,  2.95s/it]\u001b[A\n",
            " 90% 9/10 [00:24<00:02,  2.97s/it]\u001b[A[2025-01-02 13:55:54,220] [INFO] [accelerate.accelerator.gather_for_metrics:2509] [PID:2106] The used dataset had no length, returning gathered tensors. You should drop the remainder yourself.\n",
            "\n",
            "                                       \n",
            "\u001b[A{'eval_loss': 0.633816123008728, 'eval_runtime': 31.0024, 'eval_samples_per_second': 0.645, 'eval_steps_per_second': 0.323, 'epoch': 0.46}\n",
            " 15% 112/723 [54:45<4:20:24, 25.57s/it]\n",
            "100% 10/10 [00:29<00:00,  2.98s/it]\u001b[A\n",
            "{'loss': 0.4129, 'grad_norm': 0.12904559075832367, 'learning_rate': 0.0001898772583735776, 'epoch': 0.47}\n",
            "{'loss': 0.4239, 'grad_norm': 0.16067636013031006, 'learning_rate': 0.00018968321378563433, 'epoch': 0.47}\n",
            "{'loss': 0.6043, 'grad_norm': 0.1614149510860443, 'learning_rate': 0.0001894874280679026, 'epoch': 0.48}\n",
            "{'loss': 0.523, 'grad_norm': 0.15850026905536652, 'learning_rate': 0.00018928990502140963, 'epoch': 0.48}\n",
            "{'loss': 0.6249, 'grad_norm': 0.15790534019470215, 'learning_rate': 0.00018909064848091168, 'epoch': 0.48}\n",
            "{'loss': 0.5012, 'grad_norm': 0.17058630287647247, 'learning_rate': 0.00018888966231481936, 'epoch': 0.49}\n",
            "{'loss': 0.3975, 'grad_norm': 0.15669777989387512, 'learning_rate': 0.0001886869504251226, 'epoch': 0.49}\n",
            "{'loss': 0.3717, 'grad_norm': 0.146943137049675, 'learning_rate': 0.00018848251674731507, 'epoch': 0.5}\n",
            " 17% 120/723 [58:00<4:21:27, 26.02s/it]\n",
            "  0% 0/10 [00:00<?, ?it/s]\u001b[A\n",
            " 20% 2/10 [00:03<00:12,  1.60s/it]\u001b[A\n",
            " 30% 3/10 [00:06<00:16,  2.29s/it]\u001b[A\n",
            " 40% 4/10 [00:09<00:15,  2.63s/it]\u001b[A\n",
            " 50% 5/10 [00:12<00:13,  2.79s/it]\u001b[A\n",
            " 60% 6/10 [00:15<00:11,  2.86s/it]\u001b[A\n",
            " 70% 7/10 [00:18<00:08,  2.91s/it]\u001b[A\n",
            " 80% 8/10 [00:21<00:05,  2.94s/it]\u001b[A\n",
            " 90% 9/10 [00:24<00:02,  2.97s/it]\u001b[A[2025-01-02 13:59:40,885] [INFO] [accelerate.accelerator.gather_for_metrics:2509] [PID:2106] The used dataset had no length, returning gathered tensors. You should drop the remainder yourself.\n",
            "\n",
            "                                       \n",
            "\u001b[A{'eval_loss': 0.6327757835388184, 'eval_runtime': 30.9654, 'eval_samples_per_second': 0.646, 'eval_steps_per_second': 0.323, 'epoch': 0.5}\n",
            " 17% 120/723 [58:31<4:21:27, 26.02s/it]\n",
            "100% 10/10 [00:29<00:00,  2.98s/it]\u001b[A\n",
            "{'loss': 0.5601, 'grad_norm': 0.15768015384674072, 'learning_rate': 0.0001882763652503174, 'epoch': 0.5}\n",
            "{'loss': 0.6088, 'grad_norm': 0.16138140857219696, 'learning_rate': 0.00018806849993640047, 'epoch': 0.51}\n",
            "{'loss': 0.4688, 'grad_norm': 0.15813897550106049, 'learning_rate': 0.00018785892484110756, 'epoch': 0.51}\n",
            "{'loss': 0.4689, 'grad_norm': 0.1653914749622345, 'learning_rate': 0.00018764764403317598, 'epoch': 0.51}\n",
            "{'loss': 0.3585, 'grad_norm': 0.1453080028295517, 'learning_rate': 0.00018743466161445823, 'epoch': 0.52}\n",
            "{'loss': 0.4327, 'grad_norm': 0.14738991856575012, 'learning_rate': 0.0001872199817198421, 'epoch': 0.52}\n",
            "{'loss': 0.5457, 'grad_norm': 0.17621873319149017, 'learning_rate': 0.00018700360851717075, 'epoch': 0.53}\n",
            "{'loss': 0.6909, 'grad_norm': 0.18399161100387573, 'learning_rate': 0.0001867855462071614, 'epoch': 0.53}\n",
            " 18% 128/723 [1:01:59<4:13:41, 25.58s/it]\n",
            "  0% 0/10 [00:00<?, ?it/s]\u001b[A\n",
            " 20% 2/10 [00:03<00:12,  1.60s/it]\u001b[A\n",
            " 30% 3/10 [00:06<00:16,  2.30s/it]\u001b[A\n",
            " 40% 4/10 [00:09<00:15,  2.63s/it]\u001b[A\n",
            " 50% 5/10 [00:12<00:13,  2.80s/it]\u001b[A\n",
            " 60% 6/10 [00:15<00:11,  2.87s/it]\u001b[A\n",
            " 70% 7/10 [00:18<00:08,  2.91s/it]\u001b[A\n",
            " 80% 8/10 [00:21<00:05,  2.95s/it]\u001b[A\n",
            " 90% 9/10 [00:24<00:02,  2.97s/it]\u001b[A[2025-01-02 14:03:39,623] [INFO] [accelerate.accelerator.gather_for_metrics:2509] [PID:2106] The used dataset had no length, returning gathered tensors. You should drop the remainder yourself.\n",
            "\n",
            "                                         \n",
            "\u001b[A{'eval_loss': 0.6328107118606567, 'eval_runtime': 30.9854, 'eval_samples_per_second': 0.645, 'eval_steps_per_second': 0.323, 'epoch': 0.53}\n",
            " 18% 128/723 [1:02:30<4:13:41, 25.58s/it]\n",
            "100% 10/10 [00:29<00:00,  2.98s/it]\u001b[A\n",
            "{'loss': 0.3099, 'grad_norm': 0.12575390934944153, 'learning_rate': 0.0001865657990233241, 'epoch': 0.53}\n",
            "{'loss': 0.6741, 'grad_norm': 0.19602419435977936, 'learning_rate': 0.00018634437123187937, 'epoch': 0.54}\n",
            "{'loss': 0.4589, 'grad_norm': 0.14021429419517517, 'learning_rate': 0.00018612126713167542, 'epoch': 0.54}\n",
            "{'loss': 0.6025, 'grad_norm': 0.20637184381484985, 'learning_rate': 0.00018589649105410475, 'epoch': 0.55}\n",
            "{'loss': 0.458, 'grad_norm': 0.1578948050737381, 'learning_rate': 0.0001856700473630199, 'epoch': 0.55}\n",
            "{'loss': 0.5437, 'grad_norm': 0.18081635236740112, 'learning_rate': 0.00018544194045464886, 'epoch': 0.55}\n",
            "{'loss': 0.5784, 'grad_norm': 0.1660473346710205, 'learning_rate': 0.00018521217475750973, 'epoch': 0.56}\n",
            "{'loss': 0.6435, 'grad_norm': 0.15478911995887756, 'learning_rate': 0.00018498075473232469, 'epoch': 0.56}\n",
            " 19% 136/723 [1:05:53<4:21:23, 26.72s/it]\n",
            "  0% 0/10 [00:00<?, ?it/s]\u001b[A\n",
            " 20% 2/10 [00:03<00:12,  1.60s/it]\u001b[A\n",
            " 30% 3/10 [00:06<00:16,  2.29s/it]\u001b[A\n",
            " 40% 4/10 [00:09<00:15,  2.63s/it]\u001b[A\n",
            " 50% 5/10 [00:12<00:13,  2.80s/it]\u001b[A\n",
            " 60% 6/10 [00:15<00:11,  2.86s/it]\u001b[A\n",
            " 70% 7/10 [00:18<00:08,  2.91s/it]\u001b[A\n",
            " 80% 8/10 [00:21<00:05,  2.94s/it]\u001b[A\n",
            " 90% 9/10 [00:24<00:02,  2.97s/it]\u001b[A[2025-01-02 14:07:33,374] [INFO] [accelerate.accelerator.gather_for_metrics:2509] [PID:2106] The used dataset had no length, returning gathered tensors. You should drop the remainder yourself.\n",
            "\n",
            "                                         \n",
            "\u001b[A{'eval_loss': 0.6313267946243286, 'eval_runtime': 30.9374, 'eval_samples_per_second': 0.646, 'eval_steps_per_second': 0.323, 'epoch': 0.56}\n",
            " 19% 136/723 [1:06:24<4:21:23, 26.72s/it]\n",
            "100% 10/10 [00:28<00:00,  2.98s/it]\u001b[A\n",
            "{'loss': 0.3847, 'grad_norm': 0.15028592944145203, 'learning_rate': 0.00018474768487193333, 'epoch': 0.57}\n",
            "{'loss': 0.5392, 'grad_norm': 0.15695473551750183, 'learning_rate': 0.00018451296970120566, 'epoch': 0.57}\n",
            "{'loss': 0.3379, 'grad_norm': 0.19090212881565094, 'learning_rate': 0.00018427661377695397, 'epoch': 0.58}\n",
            "{'loss': 0.6786, 'grad_norm': 0.16414318978786469, 'learning_rate': 0.00018403862168784457, 'epoch': 0.58}\n",
            "{'loss': 0.5901, 'grad_norm': 0.1543651819229126, 'learning_rate': 0.00018379899805430862, 'epoch': 0.58}\n",
            "{'loss': 0.5773, 'grad_norm': 0.17544874548912048, 'learning_rate': 0.00018355774752845246, 'epoch': 0.59}\n",
            "{'loss': 0.571, 'grad_norm': 0.18718750774860382, 'learning_rate': 0.00018331487479396729, 'epoch': 0.59}\n",
            "{'loss': 0.5845, 'grad_norm': 0.17007271945476532, 'learning_rate': 0.0001830703845660381, 'epoch': 0.6}\n",
            " 20% 144/723 [1:09:50<4:15:17, 26.46s/it]\n",
            "  0% 0/10 [00:00<?, ?it/s]\u001b[A\n",
            " 20% 2/10 [00:03<00:12,  1.60s/it]\u001b[A\n",
            " 30% 3/10 [00:06<00:16,  2.30s/it]\u001b[A\n",
            " 40% 4/10 [00:09<00:15,  2.63s/it]\u001b[A\n",
            " 50% 5/10 [00:12<00:13,  2.80s/it]\u001b[A\n",
            " 60% 6/10 [00:15<00:11,  2.87s/it]\u001b[A\n",
            " 70% 7/10 [00:18<00:08,  2.92s/it]\u001b[A\n",
            " 80% 8/10 [00:21<00:05,  2.95s/it]\u001b[A\n",
            " 90% 9/10 [00:24<00:02,  2.97s/it]\u001b[A[2025-01-02 14:11:30,736] [INFO] [accelerate.accelerator.gather_for_metrics:2509] [PID:2106] The used dataset had no length, returning gathered tensors. You should drop the remainder yourself.\n",
            "\n",
            "                                         \n",
            "\u001b[A{'eval_loss': 0.629780650138855, 'eval_runtime': 30.9974, 'eval_samples_per_second': 0.645, 'eval_steps_per_second': 0.323, 'epoch': 0.6}\n",
            " 20% 144/723 [1:10:21<4:15:17, 26.46s/it]\n",
            "100% 10/10 [00:29<00:00,  2.98s/it]\u001b[A\n",
            "{'loss': 0.6207, 'grad_norm': 0.18773922324180603, 'learning_rate': 0.00018282428159125248, 'epoch': 0.6}\n",
            "{'loss': 0.4508, 'grad_norm': 0.143706277012825, 'learning_rate': 0.00018257657064750808, 'epoch': 0.6}\n",
            "{'loss': 0.8251, 'grad_norm': 0.19361193478107452, 'learning_rate': 0.00018232725654392, 'epoch': 0.61}\n",
            "{'loss': 0.3656, 'grad_norm': 0.17180386185646057, 'learning_rate': 0.00018207634412072764, 'epoch': 0.61}\n",
            "{'loss': 0.5071, 'grad_norm': 0.14708152413368225, 'learning_rate': 0.0001818238382492003, 'epoch': 0.62}\n",
            "{'loss': 0.4687, 'grad_norm': 0.16508421301841736, 'learning_rate': 0.0001815697438315429, 'epoch': 0.62}\n",
            "{'loss': 0.4047, 'grad_norm': 0.15168024599552155, 'learning_rate': 0.00018131406580080084, 'epoch': 0.63}\n",
            "{'loss': 0.3429, 'grad_norm': 0.13587258756160736, 'learning_rate': 0.00018105680912076406, 'epoch': 0.63}\n",
            " 21% 152/723 [1:13:36<4:00:38, 25.29s/it]\n",
            "  0% 0/10 [00:00<?, ?it/s]\u001b[A\n",
            " 20% 2/10 [00:03<00:12,  1.60s/it]\u001b[A\n",
            " 30% 3/10 [00:06<00:16,  2.31s/it]\u001b[A\n",
            " 40% 4/10 [00:09<00:15,  2.64s/it]\u001b[A\n",
            " 50% 5/10 [00:12<00:14,  2.80s/it]\u001b[A\n",
            " 60% 6/10 [00:15<00:11,  2.87s/it]\u001b[A\n",
            " 70% 7/10 [00:18<00:08,  2.92s/it]\u001b[A\n",
            " 80% 8/10 [00:21<00:05,  2.95s/it]\u001b[A\n",
            " 90% 9/10 [00:24<00:02,  2.97s/it]\u001b[A[2025-01-02 14:15:16,902] [INFO] [accelerate.accelerator.gather_for_metrics:2509] [PID:2106] The used dataset had no length, returning gathered tensors. You should drop the remainder yourself.\n",
            "\n",
            "                                         \n",
            "\u001b[A{'eval_loss': 0.6293768882751465, 'eval_runtime': 31.0107, 'eval_samples_per_second': 0.645, 'eval_steps_per_second': 0.322, 'epoch': 0.63}\n",
            " 21% 152/723 [1:14:07<4:00:38, 25.29s/it]\n",
            "100% 10/10 [00:29<00:00,  2.98s/it]\u001b[A\n",
            "{'loss': 0.4824, 'grad_norm': 0.170584037899971, 'learning_rate': 0.00018079797878587078, 'epoch': 0.63}\n",
            "{'loss': 0.4277, 'grad_norm': 0.14652301371097565, 'learning_rate': 0.0001805375798211104, 'epoch': 0.64}\n",
            "{'loss': 0.5082, 'grad_norm': 0.17541086673736572, 'learning_rate': 0.00018027561728192626, 'epoch': 0.64}\n",
            "{'loss': 0.4471, 'grad_norm': 0.14335158467292786, 'learning_rate': 0.00018001209625411705, 'epoch': 0.65}\n",
            "{'loss': 0.615, 'grad_norm': 0.17033329606056213, 'learning_rate': 0.00017974702185373845, 'epoch': 0.65}\n",
            "{'loss': 0.6738, 'grad_norm': 0.17641358077526093, 'learning_rate': 0.00017948039922700362, 'epoch': 0.65}\n",
            "{'loss': 0.5498, 'grad_norm': 0.1917349100112915, 'learning_rate': 0.0001792122335501833, 'epoch': 0.66}\n",
            "{'loss': 0.7318, 'grad_norm': 0.16779395937919617, 'learning_rate': 0.00017894253002950542, 'epoch': 0.66}\n",
            " 22% 160/723 [1:17:29<4:07:44, 26.40s/it]\n",
            "  0% 0/10 [00:00<?, ?it/s]\u001b[A\n",
            " 20% 2/10 [00:03<00:12,  1.60s/it]\u001b[A\n",
            " 30% 3/10 [00:06<00:16,  2.30s/it]\u001b[A\n",
            " 40% 4/10 [00:09<00:15,  2.63s/it]\u001b[A\n",
            " 50% 5/10 [00:12<00:14,  2.80s/it]\u001b[A\n",
            " 60% 6/10 [00:15<00:11,  2.87s/it]\u001b[A\n",
            " 70% 7/10 [00:18<00:08,  2.92s/it]\u001b[A\n",
            " 80% 8/10 [00:21<00:05,  2.95s/it]\u001b[A\n",
            " 90% 9/10 [00:24<00:02,  2.97s/it]\u001b[A[2025-01-02 14:19:09,978] [INFO] [accelerate.accelerator.gather_for_metrics:2509] [PID:2106] The used dataset had no length, returning gathered tensors. You should drop the remainder yourself.\n",
            "\n",
            "                                         \n",
            "\u001b[A{'eval_loss': 0.6275040507316589, 'eval_runtime': 31.0127, 'eval_samples_per_second': 0.645, 'eval_steps_per_second': 0.322, 'epoch': 0.66}\n",
            " 22% 160/723 [1:18:00<4:07:44, 26.40s/it]\n",
            "100% 10/10 [00:29<00:00,  2.98s/it]\u001b[A\n",
            "{'loss': 0.605, 'grad_norm': 0.1697687953710556, 'learning_rate': 0.00017867129390105384, 'epoch': 0.67}\n",
            "{'loss': 0.5466, 'grad_norm': 0.1521749645471573, 'learning_rate': 0.00017839853043066691, 'epoch': 0.67}\n",
            " 22% 162/723 [1:18:52<5:02:32, 32.36s/it]"
          ]
        }
      ],
      "source": [
        "!accelerate launch -m axolotl.cli.train /content/tinyllama.yaml"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "LpQTBSvdAG9R"
      },
      "source": [
        "Predict with trained model"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "fFPYRGQmAG9R"
      },
      "outputs": [],
      "source": [
        "!accelerate launch -m axolotl.cli.inference /content/tinyllama.yaml \\\n",
        "    --lora_model_dir=\"./outputs/lora-out\" --gradio"
      ]
    }
  ],
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "gpuType": "T4",
      "provenance": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python",
      "version": "3.9.6"
    },
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "b106e1468b7d4b1884d43c00d67fbcc5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "VBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "VBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "VBoxView",
            "box_style": "",
            "children": [],
            "layout": "IPY_MODEL_d7e7e44345ea447e9f67fe97eeec0bcb"
          }
        },
        "05e8461f56c04927b7550d323ec8983f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_7a7b319056cb4839992465c4a8e12b33",
            "placeholder": "​",
            "style": "IPY_MODEL_7e9cbdc5ccc3408f9af8398f8cffdca1",
            "value": "<center> <img\nsrc=https://huggingface.co/front/assets/huggingface_logo-noborder.svg\nalt='Hugging Face'> <br> Copy a token from <a\nhref=\"https://huggingface.co/settings/tokens\" target=\"_blank\">your Hugging Face\ntokens page</a> and paste it below. <br> Immediately click login after copying\nyour token or it might be stored in plain text in this notebook file. </center>"
          }
        },
        "7d065ea3a7e94cfbb552681eb8fb08a9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "PasswordModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "PasswordModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "PasswordView",
            "continuous_update": true,
            "description": "Token:",
            "description_tooltip": null,
            "disabled": false,
            "layout": "IPY_MODEL_8032f2a7e1774260841468647cad3cc9",
            "placeholder": "​",
            "style": "IPY_MODEL_2365fbc6e3304b2ea845c8c5b7c0ee9b",
            "value": ""
          }
        },
        "2bd3022ee06f41f5b4aa02f8bd3bb29a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "CheckboxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "CheckboxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "CheckboxView",
            "description": "Add token as git credential?",
            "description_tooltip": null,
            "disabled": false,
            "indent": true,
            "layout": "IPY_MODEL_b399b473b43b4abfb5ae1ad0af2fb691",
            "style": "IPY_MODEL_4500924c0fc243af9edc9d294e698f9a",
            "value": true
          }
        },
        "5d217568638d4f1895170a29856e0db8": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ButtonModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ButtonModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ButtonView",
            "button_style": "",
            "description": "Login",
            "disabled": false,
            "icon": "",
            "layout": "IPY_MODEL_bd1910d24b9740f88a3c1b02d2360a0d",
            "style": "IPY_MODEL_fce095a07d0f4b4faf1ff15f351e8828",
            "tooltip": ""
          }
        },
        "2aa1012f20274906b534d9e11f868f74": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_6406cc0b52c54d63bbba9768259c4dfa",
            "placeholder": "​",
            "style": "IPY_MODEL_25bfa563cb4c452487d48212b821c503",
            "value": "\n<b>Pro Tip:</b> If you don't already have one, you can create a dedicated\n'notebooks' token with 'write' access, that you can then easily reuse for all\nnotebooks. </center>"
          }
        },
        "d7e7e44345ea447e9f67fe97eeec0bcb": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": "center",
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": "flex",
            "flex": null,
            "flex_flow": "column",
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": "50%"
          }
        },
        "7a7b319056cb4839992465c4a8e12b33": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "7e9cbdc5ccc3408f9af8398f8cffdca1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "8032f2a7e1774260841468647cad3cc9": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "2365fbc6e3304b2ea845c8c5b7c0ee9b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "b399b473b43b4abfb5ae1ad0af2fb691": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "4500924c0fc243af9edc9d294e698f9a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "bd1910d24b9740f88a3c1b02d2360a0d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "fce095a07d0f4b4faf1ff15f351e8828": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ButtonStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ButtonStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "button_color": null,
            "font_weight": ""
          }
        },
        "6406cc0b52c54d63bbba9768259c4dfa": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "25bfa563cb4c452487d48212b821c503": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "be1dbf5ef425448aa9f0b5b2a2d771c0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "LabelModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "LabelModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "LabelView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_1c619dc43d7447788b3a11863f67283a",
            "placeholder": "​",
            "style": "IPY_MODEL_7016d2e7cf734b9e8ee09b39c96ae483",
            "value": "Connecting..."
          }
        },
        "1c619dc43d7447788b3a11863f67283a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "7016d2e7cf734b9e8ee09b39c96ae483": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        }
      }
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}