Vision Transformer.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "DLm8nlr0hUlN"
   },
   "source": [
    "# CIFAR-10 Vision Transformer"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "UzEW0MP3jb0-"
   },
   "source": [
    "Training a ViT model on CIFAR-10 dataset and using a pre-trained ViT for improving accuracy."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "id": "77p31ePZhP0b"
   },
   "outputs": [],
   "source": [
    "import torch\n",
    "import torch.nn as nn\n",
    "import torchvision.transforms as transforms\n",
    "from torch.utils.data import DataLoader\n",
    "from torchvision.datasets import CIFAR10\n",
    "from transformers import ViTModel\n",
    "from torchvision import models"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "QwkxIMQunJ5Q"
   },
   "source": [
    "## Implementation of ViT"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "HVsiTbdbhcqN"
   },
   "source": [
    "Multi-head attention module"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "leTYxqjIgEuA"
   },
   "outputs": [],
   "source": [
    "class MultiheadAttentionEinsum(nn.Module):\n",
    "    def __init__(self, embedding_dim, num_heads):\n",
    "        super(MultiheadAttentionEinsum, self).__init__()\n",
    "        self.num_heads = num_heads\n",
    "        self.head_dim = embedding_dim // num_heads\n",
    "        assert embedding_dim % num_heads == 0\n",
    "        self.q_linear = nn.Linear(embedding_dim, embedding_dim)\n",
    "        self.k_linear = nn.Linear(embedding_dim, embedding_dim)\n",
    "        self.v_linear = nn.Linear(embedding_dim, embedding_dim)\n",
    "        self.fc_out = nn.Linear(embedding_dim, embedding_dim)\n",
    "\n",
    "    def forward(self, query, key, value):\n",
    "        batch_size = query.shape[0]\n",
    "        # Linear projections\n",
    "        Q = self.q_linear(query)\n",
    "        K = self.k_linear(key)\n",
    "        V = self.v_linear(value)\n",
    "        Q = Q.view(batch_size, -1, self.num_heads, self.head_dim).transpose(1, 2)\n",
    "        K = K.view(batch_size, -1, self.num_heads, self.head_dim).transpose(1, 2)\n",
    "        V = V.view(batch_size, -1, self.num_heads, self.head_dim).transpose(1, 2)\n",
    "        # Scaled dot-product attention\n",
    "        scores = torch.einsum('bnqd,bnkd->bnqk', Q, K) / (self.head_dim ** 0.5)\n",
    "        attention = torch.softmax(scores, dim=-1)\n",
    "        # Attended values\n",
    "        attended_values = torch.einsum('bnqk,bnvd->bnqd', attention, V).transpose(1, 2)\n",
    "        attended_values = attended_values.reshape(batch_size, -1, self.num_heads * self.head_dim)\n",
    "        # Final linear projection\n",
    "        out = self.fc_out(attended_values)\n",
    "        return out"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "WCsgH6ifh48h"
   },
   "source": [
    "Encoder layer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "HuZiPWITgcl-"
   },
   "outputs": [],
   "source": [
    "class TransformerEncoderLayer(nn.Module):\n",
    "    def __init__(self, embedding_dim, num_heads):\n",
    "        super(TransformerEncoderLayer, self).__init__()\n",
    "        self.multihead_attention = MultiheadAttentionEinsum(embed_dim=embedding_dim, num_heads=num_heads)\n",
    "        self.feed_forward = nn.Sequential(\n",
    "            nn.Linear(embedding_dim, 2048),\n",
    "            nn.ReLU(),\n",
    "            nn.Linear(2048, embedding_dim)\n",
    "        )\n",
    "        self.layer_norm1 = nn.LayerNorm(embedding_dim)\n",
    "        self.layer_norm2 = nn.LayerNorm(embedding_dim)\n",
    "\n",
    "    def forward(self, x):\n",
    "        residual = x\n",
    "        x = self.layer_norm1(x)\n",
    "        x = x.permute(1, 0, 2)\n",
    "        attn_output = self.multihead_attention(x, x, x)[0]\n",
    "        x = attn_output + residual\n",
    "        x = x.permute(1, 0, 2)\n",
    "        residual = x\n",
    "        x = self.layer_norm2(x)\n",
    "        x = self.feed_forward(x)\n",
    "        x = x + residual\n",
    "        return x"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "klWsZF1uiFiN"
   },
   "source": [
    "The whole model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "UXIw0dhkmYzb"
   },
   "outputs": [],
   "source": [
    "class VisionTransformer(nn.Module):\n",
    "    def __init__(self, num_classes, patch_size, embedding_dim, num_heads, num_layers):\n",
    "        super(VisionTransformer, self).__init__()\n",
    "        self.patch_embedding = nn.Conv2d(3, embedding_dim, kernel_size=patch_size, stride=patch_size)\n",
    "        self.positional_encoding = nn.Parameter(torch.randn(1, 14 * 14 + 1, embedding_dim))\n",
    "        self.transformer_layers = nn.ModuleList([\n",
    "            nn.TransformerEncoderLayer(d_model=embedding_dim, nhead=num_heads) for _ in range(num_layers)\n",
    "        ])\n",
    "        self.fc = nn.Linear(embedding_dim, num_classes)\n",
    "\n",
    "    def forward(self, x):\n",
    "        batch_size = x.size(0)\n",
    "        x = self.patch_embedding(x)\n",
    "        x = x.flatten(2).transpose(1, 2)\n",
    "        x = torch.cat((x, self.positional_encoding.repeat(batch_size, 1, 1)), dim=1)\n",
    "        for layer in self.transformer_layers:\n",
    "            x = layer(x)\n",
    "        x = x.mean(dim=1)\n",
    "        x = self.fc(x)\n",
    "        return x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "NczLNxHriKet"
   },
   "outputs": [],
   "source": [
    "device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "Kn-fVYVNiMw0"
   },
   "source": [
    "Hyperparameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "4CWeWiUyiPa9"
   },
   "outputs": [],
   "source": [
    "num_epochs = 10\n",
    "batch_size = 64\n",
    "learning_rate = 0.001\n",
    "num_classes = 10\n",
    "patch_size = 16\n",
    "embedding_dim = 128\n",
    "num_heads = 8\n",
    "num_layers = 3"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "i2NGt13JiRB0"
   },
   "source": [
    "Loading and preprocessing dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "95mieys0iT3A"
   },
   "outputs": [],
   "source": [
    "transform = transforms.Compose([\n",
    "    transforms.Resize((224, 224)),\n",
    "    transforms.ToTensor(),\n",
    "    transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))\n",
    "])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "Hf_qf6VBgnhS",
    "outputId": "e1227286-5d39-4b3a-f43a-5837d53d8bf5"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Files already downloaded and verified\n",
      "Files already downloaded and verified\n"
     ]
    }
   ],
   "source": [
    "train_dataset = CIFAR10(root='./data', train=True, download=True, transform=transform)\n",
    "test_dataset = CIFAR10(root='./data', train=False, download=True, transform=transform)\n",
    "\n",
    "train_loader = DataLoader(dataset=train_dataset, batch_size=batch_size, shuffle=True)\n",
    "test_loader = DataLoader(dataset=test_dataset, batch_size=batch_size, shuffle=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "PKByGk6nidnD"
   },
   "source": [
    "Building the model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "Q2p2ytdgifrS"
   },
   "outputs": [],
   "source": [
    "model = VisionTransformer(num_classes, patch_size, embedding_dim, num_heads, num_layers).to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "JDkxAxM-iiF_"
   },
   "outputs": [],
   "source": [
    "criterion = nn.CrossEntropyLoss()\n",
    "optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "CkOBUTKtis6n"
   },
   "source": [
    "Training loop"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "collapsed": true,
    "id": "Q1iPe3XBg8_j",
    "outputId": "ad2b68e7-9bdb-41d2-f078-032638c6ece8"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch [1/10], Step [100/782], Loss: 2.1743\n",
      "Epoch [1/10], Step [200/782], Loss: 2.2628\n",
      "Epoch [1/10], Step [300/782], Loss: 2.0070\n",
      "Epoch [1/10], Step [400/782], Loss: 2.0852\n",
      "Epoch [1/10], Step [500/782], Loss: 2.0053\n",
      "Epoch [1/10], Step [600/782], Loss: 2.0632\n",
      "Epoch [1/10], Step [700/782], Loss: 1.8688\n",
      "Epoch [2/10], Step [100/782], Loss: 1.9895\n",
      "Epoch [2/10], Step [200/782], Loss: 2.0050\n",
      "Epoch [2/10], Step [300/782], Loss: 2.1289\n",
      "Epoch [2/10], Step [400/782], Loss: 2.0359\n",
      "Epoch [2/10], Step [500/782], Loss: 2.0502\n",
      "Epoch [2/10], Step [600/782], Loss: 1.9462\n",
      "Epoch [2/10], Step [700/782], Loss: 2.0446\n",
      "Epoch [3/10], Step [100/782], Loss: 2.0215\n",
      "Epoch [3/10], Step [200/782], Loss: 2.0119\n",
      "Epoch [3/10], Step [300/782], Loss: 1.9910\n",
      "Epoch [3/10], Step [400/782], Loss: 2.0592\n",
      "Epoch [3/10], Step [500/782], Loss: 2.0573\n",
      "Epoch [3/10], Step [600/782], Loss: 1.9202\n",
      "Epoch [3/10], Step [700/782], Loss: 2.0267\n",
      "Epoch [4/10], Step [100/782], Loss: 2.0055\n",
      "Epoch [4/10], Step [200/782], Loss: 2.1177\n",
      "Epoch [4/10], Step [300/782], Loss: 2.2351\n",
      "Epoch [4/10], Step [400/782], Loss: 2.0324\n",
      "Epoch [4/10], Step [500/782], Loss: 2.0808\n",
      "Epoch [4/10], Step [600/782], Loss: 2.0480\n",
      "Epoch [4/10], Step [700/782], Loss: 2.0805\n",
      "Epoch [5/10], Step [100/782], Loss: 2.0813\n",
      "Epoch [5/10], Step [200/782], Loss: 2.1287\n",
      "Epoch [5/10], Step [300/782], Loss: 1.9806\n",
      "Epoch [5/10], Step [400/782], Loss: 1.9849\n",
      "Epoch [5/10], Step [500/782], Loss: 2.0268\n",
      "Epoch [5/10], Step [600/782], Loss: 1.9493\n",
      "Epoch [5/10], Step [700/782], Loss: 2.0534\n",
      "Epoch [6/10], Step [100/782], Loss: 2.0899\n",
      "Epoch [6/10], Step [200/782], Loss: 1.9105\n",
      "Epoch [6/10], Step [300/782], Loss: 2.0344\n",
      "Epoch [6/10], Step [400/782], Loss: 1.9829\n",
      "Epoch [6/10], Step [500/782], Loss: 1.9103\n",
      "Epoch [6/10], Step [600/782], Loss: 2.0154\n",
      "Epoch [6/10], Step [700/782], Loss: 2.0977\n",
      "Epoch [7/10], Step [100/782], Loss: 2.0308\n",
      "Epoch [7/10], Step [200/782], Loss: 2.0660\n",
      "Epoch [7/10], Step [300/782], Loss: 1.8006\n",
      "Epoch [7/10], Step [400/782], Loss: 1.8239\n",
      "Epoch [7/10], Step [500/782], Loss: 1.9274\n",
      "Epoch [7/10], Step [600/782], Loss: 2.0221\n",
      "Epoch [7/10], Step [700/782], Loss: 1.9716\n",
      "Epoch [8/10], Step [100/782], Loss: 1.8380\n",
      "Epoch [8/10], Step [200/782], Loss: 1.6963\n",
      "Epoch [8/10], Step [300/782], Loss: 2.0507\n",
      "Epoch [8/10], Step [400/782], Loss: 1.7727\n",
      "Epoch [8/10], Step [500/782], Loss: 1.6625\n",
      "Epoch [8/10], Step [600/782], Loss: 1.8752\n",
      "Epoch [8/10], Step [700/782], Loss: 1.7577\n",
      "Epoch [9/10], Step [100/782], Loss: 1.8638\n",
      "Epoch [9/10], Step [200/782], Loss: 1.8020\n",
      "Epoch [9/10], Step [300/782], Loss: 1.9674\n",
      "Epoch [9/10], Step [400/782], Loss: 1.7082\n",
      "Epoch [9/10], Step [500/782], Loss: 1.6435\n",
      "Epoch [9/10], Step [600/782], Loss: 1.8944\n",
      "Epoch [9/10], Step [700/782], Loss: 1.9130\n",
      "Epoch [10/10], Step [100/782], Loss: 1.7538\n",
      "Epoch [10/10], Step [200/782], Loss: 1.8173\n",
      "Epoch [10/10], Step [300/782], Loss: 1.5487\n",
      "Epoch [10/10], Step [400/782], Loss: 1.9728\n",
      "Epoch [10/10], Step [500/782], Loss: 1.4453\n",
      "Epoch [10/10], Step [600/782], Loss: 1.6835\n",
      "Epoch [10/10], Step [700/782], Loss: 1.6234\n"
     ]
    }
   ],
   "source": [
    "total_steps = len(train_loader)\n",
    "for epoch in range(num_epochs):\n",
    "    model.train()\n",
    "    for i, (images, labels) in enumerate(train_loader):\n",
    "        images = images.to(device)\n",
    "        labels = labels.to(device)\n",
    "        outputs = model(images)\n",
    "        loss = criterion(outputs, labels)\n",
    "        optimizer.zero_grad()\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "        if (i + 1) % 100 == 0:\n",
    "            print(f'Epoch [{epoch+1}/{num_epochs}], Step [{i+1}/{total_steps}], Loss: {loss.item():.4f}')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "HuNsN7Yki-sk"
   },
   "source": [
    "Evaluation and accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "AoG3m2IPhCoH",
    "outputId": "dff9e961-3a67-4115-f2e3-1a894af0721d"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test Accuracy of the model on the 10000 test images: 38.99%\n"
     ]
    }
   ],
   "source": [
    "model.eval()\n",
    "with torch.no_grad():\n",
    "    correct = 0\n",
    "    total = 0\n",
    "    for images, labels in test_loader:\n",
    "        images = images.to(device)\n",
    "        labels = labels.to(device)\n",
    "        outputs = model(images)\n",
    "        _, predicted = torch.max(outputs.data, 1)\n",
    "        total += labels.size(0)\n",
    "        correct += (predicted == labels).sum().item()\n",
    "    accuracy = 100 * correct / total\n",
    "    print(f'Test Accuracy of the model on the {total} test images: {accuracy:.2f}%')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "tZ_J3f4XnWV3"
   },
   "source": [
    "---\n",
    "Here we change and test hyperparameters to get better results.\n",
    "\n",
    "It is also notable that we could not run a heavy model due to colab's limitations.\n",
    "\n",
    "\n",
    "Overall changes\n",
    "* Higher value for number of epochs\n",
    "* Increased batch size and number of layers\n",
    "* Reduced embedding dim and number of heads"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "iUl0a1RcpTxb"
   },
   "outputs": [],
   "source": [
    "num_epochs = 20\n",
    "batch_size = 256\n",
    "learning_rate = 0.001\n",
    "num_classes = 10\n",
    "patch_size = 16\n",
    "embedding_dim = 64\n",
    "num_heads = 4\n",
    "num_layers = 4"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "TBTlHwqnpRWZ"
   },
   "outputs": [],
   "source": [
    "transform = transforms.Compose([\n",
    "    transforms.Resize((224, 224)),\n",
    "    transforms.ToTensor(),\n",
    "    transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))\n",
    "])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "cZzMthYJpMsZ",
    "outputId": "e5685976-ab53-469b-de14-867e1a56d5e3"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Files already downloaded and verified\n",
      "Files already downloaded and verified\n"
     ]
    }
   ],
   "source": [
    "train_dataset = CIFAR10(root='./data', train=True, download=True, transform=transform)\n",
    "test_dataset = CIFAR10(root='./data', train=False, download=True, transform=transform)\n",
    "\n",
    "train_loader = DataLoader(dataset=train_dataset, batch_size=batch_size, shuffle=True)\n",
    "test_loader = DataLoader(dataset=test_dataset, batch_size=batch_size, shuffle=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "Hizh8YWBoJF_"
   },
   "outputs": [],
   "source": [
    "model = VisionTransformer(num_classes, patch_size, embedding_dim, num_heads, num_layers).to(device)\n",
    "criterion = nn.CrossEntropyLoss()\n",
    "optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "HPBbrMx_oiPh",
    "outputId": "87441d58-5395-4ee1-d419-1c1dd281e972"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch [1/20], Step [100/196], Loss: 2.1333\n",
      "Epoch [2/20], Step [100/196], Loss: 1.9316\n",
      "Epoch [3/20], Step [100/196], Loss: 1.8236\n",
      "Epoch [4/20], Step [100/196], Loss: 1.6120\n",
      "Epoch [5/20], Step [100/196], Loss: 1.6060\n",
      "Epoch [6/20], Step [100/196], Loss: 1.4951\n",
      "Epoch [7/20], Step [100/196], Loss: 1.4209\n",
      "Epoch [8/20], Step [100/196], Loss: 1.2905\n",
      "Epoch [9/20], Step [100/196], Loss: 1.3277\n",
      "Epoch [10/20], Step [100/196], Loss: 1.2982\n",
      "Epoch [11/20], Step [100/196], Loss: 1.3345\n",
      "Epoch [12/20], Step [100/196], Loss: 1.2371\n",
      "Epoch [13/20], Step [100/196], Loss: 1.1746\n",
      "Epoch [14/20], Step [100/196], Loss: 1.1767\n",
      "Epoch [15/20], Step [100/196], Loss: 1.1174\n",
      "Epoch [16/20], Step [100/196], Loss: 0.9692\n",
      "Epoch [17/20], Step [100/196], Loss: 1.1116\n",
      "Epoch [18/20], Step [100/196], Loss: 1.1323\n",
      "Epoch [19/20], Step [100/196], Loss: 0.9683\n",
      "Epoch [20/20], Step [100/196], Loss: 1.0162\n"
     ]
    }
   ],
   "source": [
    "total_steps = len(train_loader)\n",
    "for epoch in range(num_epochs):\n",
    "    model.train()\n",
    "    for i, (images, labels) in enumerate(train_loader):\n",
    "        images = images.to(device)\n",
    "        labels = labels.to(device)\n",
    "        outputs = model(images)\n",
    "        loss = criterion(outputs, labels)\n",
    "        optimizer.zero_grad()\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "        if (i + 1) % 100 == 0:\n",
    "            print(f'Epoch [{epoch+1}/{num_epochs}], Step [{i+1}/{total_steps}], Loss: {loss.item():.4f}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "ffAhIG1RosLL",
    "outputId": "55a70fda-669c-4ee0-e246-9174c3ad0d5f"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test Accuracy of the model on the 10000 test images: 62.17%\n"
     ]
    }
   ],
   "source": [
    "model.eval()\n",
    "with torch.no_grad():\n",
    "    correct = 0\n",
    "    total = 0\n",
    "    for images, labels in test_loader:\n",
    "        images = images.to(device)\n",
    "        labels = labels.to(device)\n",
    "        outputs = model(images)\n",
    "        _, predicted = torch.max(outputs.data, 1)\n",
    "        total += labels.size(0)\n",
    "        correct += (predicted == labels).sum().item()\n",
    "    accuracy = 100 * correct / total\n",
    "    print(f'Test Accuracy of the model on the {total} test images: {accuracy:.2f}%')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "cut3Rgw-6WWw"
   },
   "source": [
    "---\n",
    "## Pre-Trained Vision Transformer\n",
    "There is also a built-in version of ViT model which was trained on ImageNet-21k at resolution 224 * 224. Here we try this model."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "id": "bYRkHiZ_w-fy"
   },
   "outputs": [],
   "source": [
    "transform = transforms.Compose([\n",
    "    transforms.Resize((224, 224)),\n",
    "    transforms.ToTensor(),\n",
    "    transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))\n",
    "])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "KhTXX-3z7CXI",
    "outputId": "d71a9f24-8a68-4d10-e0c5-8c7c289984a4"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Downloading https://www.cs.toronto.edu/~kriz/cifar-10-python.tar.gz to ./data/cifar-10-python.tar.gz\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 170498071/170498071 [00:04<00:00, 42503736.94it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Extracting ./data/cifar-10-python.tar.gz to ./data\n",
      "Files already downloaded and verified\n"
     ]
    }
   ],
   "source": [
    "train_dataset = CIFAR10(root='./data', train=True, download=True, transform=transform)\n",
    "test_dataset = CIFAR10(root='./data', train=False, download=True, transform=transform)\n",
    "\n",
    "train_loader = DataLoader(dataset=train_dataset, batch_size=256, shuffle=True)\n",
    "test_loader = DataLoader(dataset=test_dataset, batch_size=256, shuffle=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 207,
     "referenced_widgets": [
      "9a320ed20f334618badc57ea72809b00",
      "bb12c0581b564445bab0f91e2d9a098f",
      "ace7475e6fc840c2b762911fda84ff28",
      "ce8f90eb3ad24e99a3b6668718ebd0a7",
      "f5202206586e4c329bbcc39de1c94264",
      "cf616281a03046628d95aae643608d96",
      "461aba97107a4352899aeebae8c518ec",
      "98ae0a865b9948c1b6e614d543ce9210",
      "8c34bc9b82a4426095f9c919d290139f",
      "57cafc338c5c414f828ec78345ade606",
      "7508183ccb7e4e3980621ee2beff9d44",
      "dfedc00f4ce749f0b3d8048dbae94659",
      "d0260e4e3ec145d3a94f9ddd8df87f00",
      "a39e07d1a9ec4fb1a83b73b03d3c3721",
      "bd53b0fff93e4e77a3a3248e32c77b7a",
      "3d5f89633df44e9f9423de94c8dcb11b",
      "01f313aa78dd4fb9a43eeef2ac9c2f25",
      "4410ab23cd7b47a39e56d43d18aecb29",
      "0dffb96249394778817764ab7ae73f55",
      "4e4e74e14d414e249fb27163de2b7178",
      "aeddbd0347e84be29068b10d98cfa113",
      "3905d7327c6c4dcfae873cb34c41603f"
     ]
    },
    "id": "kIuGOQ4I7Iq7",
    "outputId": "5facc9d7-6d31-4036-ba5c-f61100482b94"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.10/dist-packages/huggingface_hub/utils/_token.py:89: UserWarning: \n",
      "The secret `HF_TOKEN` does not exist in your Colab secrets.\n",
      "To authenticate with the Hugging Face Hub, create a token in your settings tab (https://huggingface.co/settings/tokens), set it as secret in your Google Colab and restart your session.\n",
      "You will be able to reuse this secret in all of your notebooks.\n",
      "Please note that authentication is recommended but still optional to access public models or datasets.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9a320ed20f334618badc57ea72809b00",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "config.json:   0%|          | 0.00/502 [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "dfedc00f4ce749f0b3d8048dbae94659",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "model.safetensors:   0%|          | 0.00/346M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "vit_model = ViTModel.from_pretrained('google/vit-base-patch16-224-in21k')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "_GpmVyt17QUs"
   },
   "source": [
    "Freezing params"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "id": "4_UZFWJ37MCa"
   },
   "outputs": [],
   "source": [
    "for param in vit_model.parameters():\n",
    "      param.requires_grad = False"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "YlzeeRmX7_v_"
   },
   "source": [
    "Designing the network"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "id": "h2ffdZJr7R2s"
   },
   "outputs": [],
   "source": [
    "class ViTForImageClassification(nn.Module):\n",
    "    def __init__(self, num_labels=10):\n",
    "        super(ViTForImageClassification, self).__init__()\n",
    "        self.vit = vit_model\n",
    "        self.dropout = nn.Dropout(0.1)\n",
    "        self.classifier = nn.Linear(self.vit.config.hidden_size, num_labels)\n",
    "        self.num_labels = num_labels\n",
    "\n",
    "    def forward(self, pixel_values):\n",
    "        outputs = self.vit(pixel_values=pixel_values)\n",
    "        output = self.dropout(outputs.last_hidden_state[:, 0])\n",
    "        logits = self.classifier(output)\n",
    "        return logits"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "id": "VdmKsYAW7dtL"
   },
   "outputs": [],
   "source": [
    "device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "id": "nD05X1HP7Vym"
   },
   "outputs": [],
   "source": [
    "model = ViTForImageClassification()\n",
    "model = model.to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "id": "dJKSSMLQ7ebe"
   },
   "outputs": [],
   "source": [
    "criterion = nn.CrossEntropyLoss()\n",
    "optimizer = torch.optim.Adam(model.parameters(), lr=0.001)\n",
    "epochs = 3"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "RCyLEVHJ8HFX"
   },
   "source": [
    "Training loop"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "_uezGPfK7lRN",
    "outputId": "babd9ad8-d79e-4b08-82e4-2d68b290ff32"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch [1/3], Step [100/196], Loss: 0.3800\n",
      "Epoch [2/3], Step [100/196], Loss: 0.1560\n",
      "Epoch [3/3], Step [100/196], Loss: 0.1324\n"
     ]
    }
   ],
   "source": [
    "total_steps = len(train_loader)\n",
    "for epoch in range(epochs):\n",
    "    model.train()\n",
    "    for i, (images, labels) in enumerate(train_loader):\n",
    "        images = images.to(device)\n",
    "        labels = labels.to(device)\n",
    "        outputs = model(images)\n",
    "        loss = criterion(outputs, labels)\n",
    "        optimizer.zero_grad()\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "        if (i + 1) % 100 == 0:\n",
    "            print(f'Epoch [{epoch + 1}/{epochs}], Step [{i + 1}/{total_steps}], Loss: {loss.item():.4f}')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "X0glh_bFHs67"
   },
   "source": [
    "Even after 3 epochs, the result was amazing in comparison to the previous model."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "O4lN9fiJ8Ivc"
   },
   "source": [
    "Evaluation and test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "MQYiobNS70gY",
    "outputId": "45dedd98-9904-4349-b800-69f4582668aa"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test Accuracy of the model on the 10000 test images: 96.10%\n"
     ]
    }
   ],
   "source": [
    "model.eval()\n",
    "with torch.no_grad():\n",
    "    correct = 0\n",
    "    total = 0\n",
    "    for images, labels in test_loader:\n",
    "        images = images.to(device)\n",
    "        labels = labels.to(device)\n",
    "        outputs = model(images)\n",
    "        _, predicted = torch.max(outputs.data, 1)\n",
    "        total += labels.size(0)\n",
    "        correct += (predicted == labels).sum().item()\n",
    "    accuracy = 100 * correct / total\n",
    "    print(f'Test Accuracy of the model on the {total} test images: {accuracy:.2f}%')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "id": "rpJ8ypzzGywN"
   },
   "outputs": [],
   "source": [
    "torch.save(model.state_dict(), 'pre-trained ViT.pth')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "VDOoNCcrRik9"
   },
   "source": [
    "---\n",
    "## Pre-Trained CNNs\n",
    "Here we compare the result of the last past with one of the pre-trained CNN models."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "id": "9OohaoVBRhIM"
   },
   "outputs": [],
   "source": [
    "transform = transforms.Compose([\n",
    "    transforms.Resize((224, 224)),\n",
    "    transforms.ToTensor(),\n",
    "    transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))\n",
    "])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "ox-yd7kBSJsM",
    "outputId": "13e2d7d2-40e1-4ca4-cba3-d14fed0e2d04"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Files already downloaded and verified\n",
      "Files already downloaded and verified\n"
     ]
    }
   ],
   "source": [
    "train_dataset = CIFAR10(root='./data', train=True, download=True, transform=transform)\n",
    "test_dataset = CIFAR10(root='./data', train=False, download=True, transform=transform)\n",
    "\n",
    "train_loader = DataLoader(dataset=train_dataset, batch_size=256, shuffle=True)\n",
    "test_loader = DataLoader(dataset=test_dataset, batch_size=256, shuffle=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "vnoE37DjSe4f"
   },
   "source": [
    "Here we used ResNet18"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "b-EGE1bYSNFn",
    "outputId": "5afc83a2-4886-48b5-d009-4266d9d24c04"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.10/dist-packages/torchvision/models/_utils.py:208: UserWarning: The parameter 'pretrained' is deprecated since 0.13 and may be removed in the future, please use 'weights' instead.\n",
      "  warnings.warn(\n",
      "/usr/local/lib/python3.10/dist-packages/torchvision/models/_utils.py:223: UserWarning: Arguments other than a weight enum or `None` for 'weights' are deprecated since 0.13 and may be removed in the future. The current behavior is equivalent to passing `weights=ResNet18_Weights.IMAGENET1K_V1`. You can also use `weights=ResNet18_Weights.DEFAULT` to get the most up-to-date weights.\n",
      "  warnings.warn(msg)\n"
     ]
    }
   ],
   "source": [
    "model = models.resnet18(pretrained=True)\n",
    "num_ftrs = model.fc.in_features\n",
    "model.fc = nn.Linear(num_ftrs, 10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "id": "BxaYetuqSzSY"
   },
   "outputs": [],
   "source": [
    "device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n",
    "model = model.to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "id": "abYnRIdhSpMw"
   },
   "outputs": [],
   "source": [
    "criterion = nn.CrossEntropyLoss()\n",
    "optimizer = torch.optim.Adam(model.parameters(), lr=0.001)\n",
    "epochs = 3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "yCrDGwCPTUEL",
    "outputId": "7cedcc7e-6e66-484c-b74e-2a4cf6166c8d"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch [1/3], Step [100/196], Loss: 0.3257\n",
      "Epoch [2/3], Step [100/196], Loss: 0.1958\n",
      "Epoch [3/3], Step [100/196], Loss: 0.0993\n"
     ]
    }
   ],
   "source": [
    "total_steps = len(train_loader)\n",
    "for epoch in range(epochs):\n",
    "    model.train()\n",
    "    for i, (images, labels) in enumerate(train_loader):\n",
    "        images = images.to(device)\n",
    "        labels = labels.to(device)\n",
    "        outputs = model(images)\n",
    "        loss = criterion(outputs, labels)\n",
    "        optimizer.zero_grad()\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "        if (i + 1) % 100 == 0:\n",
    "            print(f'Epoch [{epoch + 1}/{epochs}], Step [{i + 1}/{total_steps}], Loss: {loss.item():.4f}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "JQIWktvwTWow",
    "outputId": "83d6ea27-e974-4d1d-bfa3-613e5ecf28cb"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test Accuracy of the model on the 10000 test images: 89.34%\n"
     ]
    }
   ],
   "source": [
    "model.eval()\n",
    "with torch.no_grad():\n",
    "    correct = 0\n",
    "    total = 0\n",
    "    for images, labels in test_loader:\n",
    "        images = images.to(device)\n",
    "        labels = labels.to(device)\n",
    "        outputs = model(images)\n",
    "        _, predicted = torch.max(outputs.data, 1)\n",
    "        total += labels.size(0)\n",
    "        correct += (predicted == labels).sum().item()\n",
    "    accuracy = 100 * correct / total\n",
    "    print(f'Test Accuracy of the model on the {total} test images: {accuracy:.2f}%')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "uefY8B5uXm3I"
   },
   "source": [
    "ViT was better than ResNet18 although ResNet18 had a better result on training set.\n",
    "\n",
    "This means that vision transformers are more generalized than CNNs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "id": "HbrXglsPXl7w"
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAnAAAAGRCAYAAADl444ZAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/H5lhTAAAACXBIWXMAAAsTAAALEwEAmpwYAABEOElEQVR4nO3de3zP9f//8ft7Zza2Yc4lp+Ezh2EODZPzYRmicmwlfJzyEcLIoSXxMbbM4ddqNFIfhVIUKyH6qBwTkRxDYlvMxthmr98ffby/3m3jLZvtxe16uexy2fv5fL1fr8f7ted7u+/1er5eb4thGIYAAABgGg4FXQAAAADuDAEOAADAZAhwAAAAJkOAAwAAMBkCHAAAgMkQ4AAAAEyGAAcUEq1bt1aNGjUUHR2dY//atWtVo0YNPfXUU3e1ndOnT6tGjRr6+uuv76i2iIiIXPv79++vGjVqWL9q1qypBg0aqFevXne0ndwYhqEpU6aofv36atSokX7//fe7Xuf9ZM+ePRo5cqSaN2+u+vXrKyQkRO+++64yMzMLurS/7e+MU+BBQoADChGLxaIvv/wyx774+Ph7XM2dCQwM1IoVK7RixQq9//77euONN1S8eHENGTJEBw4cuKt179u3TytWrNDzzz+vBQsWqEyZMnlUtfmtWLFC/fr1k6Ojo6ZOnaoFCxaoVatWmjVrlsaPHy+z3uqzdOnSWrFiherXr1/QpQCFklNBFwDg//j7+2vPnj06ffq0KlasaG2/evWqtm7dKl9f3wKs7ta8vLzk7+9v09aoUSMFBQVpxYoVCg8P/9vrvnTpkiSpa9eueuihh+6mzPvKzz//rFdffVVDhw7ViBEjrO2BgYGqXLmyxo8fr+DgYLVu3boAq/x7XFxcso0nAP+HI3BAIdKgQQOVLFky21G4rVu3ytvbW35+fjbt6enpmj9/vtq3b6+6deuqe/fu2rRpk80yR44cUWhoqPz9/RUcHKz9+/dn2+6ePXvUu3dv1a1bVy1atFB0dLSysrLu+vW4ubnpkUce0W+//Wb3tlq3bq3IyEh1795d9erVU7t27TRw4EBJUtu2bTVhwgRJUmJioiZMmKDmzZvL399fgwYN0tGjR63riY6OVu/evTV16lTVr19fgwYN0nfffacaNWpox44d6t69u+rUqaMnnnhChw8f1oYNG9S+fXs1aNBA//rXv5SammqzD4cPH67GjRurdu3a6tChgz788MNs21q9erXatm2rOnXqqG/fvjpy5IjN/li7dq1CQkJUr149dejQQatWrbLpX7VqlTp27KjatWurc+fO+vzzz2+5f5cvXy4vLy8NHjw4W1/Xrl317LPPyt3d3dp24MABPffccwoICFDTpk01efJkpaSkWPv79++vf//733rttdfUqFEjNW7cWHPnztWlS5c0evRo+fv7q23btjbjs3///poxY4amT5+uBg0aqHnz5po3b57Nz/T333/XSy+9pMDAQPn5+al169ZatGiRtX/16tUKCgrSggUL1KhRI3Xp0kWnTp2yOYV6+fJlhYWFqVmzZqpbt6569eql7du327zmzz77zDpu2rRpo7feesvmCGSNGjW0Zs0ajRgxQv7+/mrWrJnmz59/y30MFFYEOKAQcXBwUOvWrbVx40ab9vj4eHXo0CHb8mPHjtXixYvVr18/zZ8/X9WqVdPQoUOtIS4lJUWhoaG6du2aIiMj9fTTT2vSpEk26zh06JBCQ0Pl5eWl6OhoDRo0SLGxsZo9e/Zdv57MzEydOXNGFSpUuKNtxcbGqmvXroqIiNA777yjsLAwSdL8+fM1bNgwpaamqlevXtqzZ4/CwsIUERGhCxcuqE+fPjp79qx1Pfv27dNvv/2mhQsXasCAAdb2cePGqV+/fpo3b56SkpI0dOhQRUdHa9y4cRo3bpzi4+O1dOlSSVJqaqqeeeYZXbt2TREREVq4cKGqVKmiyZMn6/jx49Z1Hj58WLGxsRozZowiIyN19uxZa93Sn+FizJgxCggI0MKFCxUcHKyJEydaf9YrVqzQyy+/bA03zZo104svvqivvvoq1/377bffqkmTJnJxccnWZ7FYFBYWpiZNmkiS9u/fr169esnZ2VkREREaM2aMNm7cqEGDBun69evW5/3nP//RuXPnNG/ePHXt2lVvvvmmnnzySZUvX16LFi1SmTJlNGHCBF25csX6nNWrV2v//v2aM2eO+vfvrzfffFMLFiyQJGVlZWngwIE6ceKEXn31Vb311lsKDAxUVFSUvvnmG+s6kpKStGnTJkVFRWn06NGyWCw2r+e1117Tjh07NGXKFL355pvy9vbW0KFD9ccff0iS3n33XY0ePVqNGzfWggUL1L17d73xxhvZxtb06dP18MMPa9GiRercubOio6O1ZcuWXPcxUGgZAAqFVq1aGbNnzzY2b95s1KpVy/jjjz8MwzCM9PR0o2HDhsbu3buN8ePHG08++aRhGIZx8OBBw9fX11izZo3Nep555hkjJCTEMAzDiIuLM+rWrWskJSVZ+2NjYw1fX19jy5YthmEYxsiRI43OnTsbmZmZ1mU++OADw8/Pz0hMTLSpLTf9+vUz/vWvfxkZGRlGRkaGce3aNePkyZNGWFiYUaNGDWPfvn13tK2nn37aZv1btmwxfH19jVOnThmGYRjvvPOO4efnZ31sGIZx6dIlo1GjRsb06dMNwzCMefPmGb6+vsbx48ety3z77beGr6+vsWzZMmtbdHS04evra+zYscPa9uyzzxpDhgwxDMMw9u3bZ/Tp08e4dOmStf/ixYuGr6+vsXLlSpttHTlyxLrMf/7zH8PX19f6c+zatasxbNgwm9c1evRoY+bMmcb169eNwMBAY/LkyTb9Y8eOtf4sc1KvXj0jIiIi1/6bDRs2zOjQoYPNvt+xY4fh6+trfPHFF4Zh/PlzbNasmZGenm4Yxp9jz8/PzwgNDbU+Z+fOnYavr69x4MAB63MaNmxos39mzpxpNGjQwEhPTzfOnDlj9OvXzzh69Ki1PysrywgICDDeeOMNwzAMY9WqVYavr6+xbds26zKnTp2yGacdOnSw2T8XL140Zs6caZw+fdrIzMw0GjdubEyaNMnmNc+fP9/w8/Ozjn9fX19j1KhRNnU0b97cCA8Pt2sfAoUJR+CAQubRRx9VkSJFrEfRtm/friJFimSbD7Rr1y5ZLBZ17NjRpr1z5846dOiQUlNTtXv3btWuXVslSpSw9rdr185m+R07dqhZs2YyDEOZmZnKzMxUixYtlJGRod27d9td9+effy4/Pz/5+fmpTp06ateunTZt2qTw8HDVqVPnjrZVuXLlW25r165dql27ts08wWLFiqlFixbatWuXtc3JySnHOXN169a1fl+yZElJsjk97eXlZT2FWqdOHS1fvlyurq76+eeftWHDBsXExEiSMjIyrM/x8PBQ1apVrY/Lli0rSUpLS9PVq1d18OBBPfbYYzZ1zJkzR+PHj9fx48eVmJiooKAg6365sW8OHTqkixcv5rgfHBwcbI6e3cquXbvUvn17OTo6WtsCAgLk4+Njs8/+8Y9/yNnZWZLk7OwsT09P/eMf/7DZN5JsTr22aNFCxYoVsz5u06aNUlNT9csvv6h8+fJatmyZKlWqpGPHjmnjxo2aP3++MjMzbfafdOufe0BAgD744AMNGzZMK1eu1PXr1zV+/HhVqFBBx44d08WLF3N8L2RkZOiHH36wttWrV8/6vcViUZkyZWyOJgJmwUUMQCHj4uKili1b6ssvv9QTTzyh+Ph4tWvXLtsppeTkZBUrVizb6bMbgeTy5cu6dOmSvL29c+y/4eLFi4qLi1NcXFy2Ws6fP2933c2bN9eoUaMk/RksihcvrooVK9rUbe+2/lrjX126dCnHZUqUKGEzx8/Ly8smsNxw87ywG4oUKZLr9ubPn6/Y2FilpaXpoYceUuPGjSXJZn6Vm5ubzXNuvO6srCwlJydb68vJhQsXJEnDhw/PsT8xMdEanG5Wvnz5W95S5dy5cypdurQsFkuu+6xkyZI28/3udN9Iko+Pj83jG6/zxutesWKFIiMjdeHCBZUrV04NGzaUk5NTtitkb/Vzf/nll1WqVCmtWbNGGzdulJOTkzp16qRXX33Vup1SpUrluL6bX5+rq6vNMg4ODqa9UhcPNgIcUAi1a9dOEyZM0OXLl/XVV19p7ty52Zbx9PRUSkqK0tPTbUJcYmKitd/T0zPbH/gbf+xuKFasmB5//HF169Yt2zbKly9vd83Fixe3HmnLTV5uK6dwmZSUJE9PT7vXY4+PP/5YCxcu1Guvvab27dvL3d1dV69e1cqVK+1ex41QdCOo3XDs2DGlpKSoePHikqTXX39d1atXz/b8m4803uzRRx/V559/royMDOtRs5v16dNHfn5+mjdvnooXL66kpKRsy+TFPvvrmLoxL61EiRL6/vvvNXXqVI0ZM0ZPPfWUdVuBgYF3tA03NzeNGjVKo0aN0uHDh7Vu3TrFxMSoevXq1qtsb4z9G25+LwD3G06hAoVQUFCQDMPQwoULlZWVpUaNGmVbpkGDBjIMQ+vXr7dp//zzz1WrVi25ubmpUaNG+vHHH21C3NatW22Wr1+/vk6cOKE6depYv5ycnBQVFZXjH/y7kVfbatCggfbv36/Tp09b21JTU7Vt27Y8v/XEnj179Mgjj6h79+7WIHZj8r29V+p6eHioevXq2W5KGxUVpcjISFWpUkVeXl5KTEy02Tc///yzYmJi5OCQ86/qXr166cKFC3r77bez9a1evVqnT59Wp06dJP25z+Lj421Oue7cuVMJCQl3vc/++9//Kj093fr4yy+/lJeXl6pXr669e/fK1dVVgwYNsgapQ4cOKSkpye79ZxiGunXrpnfeeUeS5OvrqxdffFGPPPKIfv/9d+v+y+m94OjoaHPKHLhfcAQOKITc3d0VGBiouLg4devWLcfTgLVq1VLbtm01bdo0Xbx4UZUrV9batWv13XffWW+N0L17d8XGxmrw4MEaOXKkEhMTs33Sw5AhQ9S3b1+FhYWpc+fOSk5OVmRkpIoUKXLbuWh3Kq+21aNHDy1dulTPP/+8/vWvf8nFxUUxMTHKyspSaGhontZcu3ZtrVixQjExMfL399dPP/2k+fPny2KxKC0tze71DBkyRGPHjtWMGTP02GOPaceOHfriiy/05ptvysnJSf/85z8VFRWljIwMNWzYUIcOHVJkZKRCQkJyvMpUkqpWraoXX3xRs2fP1tGjR9W5c2c5OTlp27Zteu+999S1a1drgBsyZIj69OmjoUOHqm/fvkpISFBUVJTq1KmTbW7enUpISNALL7ygvn37av/+/Vq2bJnGjRsnBwcH1alTR1evXtWsWbPUqlUrnThx4o73n8ViUd26dbVw4UIVLVpUlSpV0vbt23X8+HFNnjxZjo6OGjZsmF5//XW5u7srKChIe/fu1aJFi9S/f/8cTz8DZkeAAwqpGxcB/PWig5vNmTNHUVFRiomJ0aVLl+Tr66tFixapVatWkv6cuxQXF6fw8HCNHTtWPj4+mjZtms1NX/39/RUbG6uoqCgNHz5c7u7uat68uV566aUcT8vdjbzaVrFixbRs2TLNmjVLL7/8siwWiwICArRixQrrLUvySo8ePXTs2DHFxcVp0aJFevjhhzVx4kR98sknNpPjb+fxxx9XZmam3nzzTb3//vuqVKmS5s6dq6CgIEnSgAED5ObmZt1O6dKl9dxzz+U6L+6GgQMHqnLlylq6dKkmTZqkq1ev6pFHHtGkSZP05JNPWperW7eulixZojlz5mjEiBEqVqyY2rVrp7Fjx8rJ6e7+FLRr107e3t4aOXKkSpQooQkTJqh///6S/jzN+9JLL2nZsmV67733VL58eYWGhurIkSN3tP/CwsLk7Oys+fPn648//lClSpU0c+ZM66nY0NBQubq6asmSJVq+fLnKlSunF1980eb2McD9xGIwexMA8Df1799fpUqVUmRkZEGXAjxQmAMHAABgMgQ4AAAAk+EUKgAAgMkU2BG4a9euKTg42Oay+pSUFI0ZM0YNGzZUixYttGTJEpvn3K4fAADgQVAgV6GmpaXpxRdf1JEjR2zaJ02apPPnz2v58uU6ceKEwsLCVLp0aQUHB9vVDwAA8CC45wFu//79Gj9+fLZbBpw5c0bx8fFau3atqlWrppo1a+rIkSOKi4tTcHDwbfsBAAAeFPc8wG3fvl0tW7bUyJEjbT5UeO/evfLy8lK1atWsbQEBAVq0aJEyMjJu238n95C6cOGysrKY+ldQSpb0UFJS6u0XBEyEcY37DWO64Dk4WOTtnf3ziaUCCHCDBg3Ksf3Ghy7fzMfHR5mZmUpMTLxtf7ly5eyuISvLIMAVMPY/7keMa9xvGNOFV6H5JIa0tDS5urratN34+Jj09PTb9t+JkiU97qJS5AUfn2IFXQKQ5xjXuN8wpguvQhPg3NzcsgWxG4/d3Nxu238nkpJS+a+iAPn4FFNCQkpBlwHkKcY17jeM6YLn4GDJ9aBTobmRb9myZZWQkGDTdv78eTk7O8vb2/u2/QAAAA+KQhPg/P39lZSUpOPHj1vbdu3apdq1a8vFxeW2/QAAAA+KQhPgKlSooFatWmn8+PH66aeftGHDBsXGxio0NNSufgAAgAdFoZkDJ0kzZ87UlClT1Lt3b3l6emrUqFHq1KmT3f0AAAAPggfys1C5iKFgMTEW9yPGNe43jOmCZ4qLGAAAAGAfAhwAAIDJEOAAAABMhgAHAABgMgQ4AAAAkyHAAQAAmEyhug8cgMLJ3dNRRV2KFnQZhR4f/J27K+lXdDn5ekGXAdw3CHAAbquoS1FZXrEUdBkwMWOqocvinmJAXuEUKgAAgMkQ4AAAAEyGAAcAAGAyBDgAAACTIcABAACYDAEOAADAZAhwAAAAJkOAAwAAMBkCHAAAgMkQ4AAAAEyGAAcAAGAyBDgAAACTIcABAACYDAEOAADAZAhwAAAAJkOAAwAAMBkCHAAAgMkQ4AAAAEyGAAcAAGAyBDgAAACTcSroAgAAuNdKujvKoWjRgi6j0PPxKVbQJRRaWVeuKOny9QLbPgEOAPDAcShaVLJYCroMmJiDYUiXUwpu+wW2ZQAAAPwtBDgAAACTIcABAACYDAEOAADAZAhwAAAAJkOAAwAAMBkCHAAAgMkQ4AAAAEyGAAcAAGAyBDgAAACTIcABAACYDAEOAADAZAhwAAAAJkOAAwAAMBkCHAAAgMkQ4AAAAEyGAAcAAGAyBDgAAACTIcABAACYDAEOAADAZAhwAAAAJkOAAwAAMJlCF+AuXbqkCRMmqEmTJgoMDNSUKVN0+fJlSVJGRobCw8PVpEkTNWnSRBEREcrKyirgigEAAO6tQhfgXnnlFZ04cUJxcXFauHChvv/+e82aNUuSNHfuXH3zzTeKiYlRZGSkPv74Y7399tsFXDEAAMC9VegC3ObNm/Xss8+qZs2a8vf3V9++ffXtt9/q2rVrev/99zVx4kTVq1dPgYGBGjNmjJYuXSrDMAq6bAAAgHum0AU4b29vffLJJ0pNTdXFixe1YcMG+fn56eDBg0pLS1NAQIB12YCAACUkJOj06dMFWDEAAMC9VegCXHh4uH744Qc1atRITZs21aVLlxQeHq5z587Jw8ND7u7u1mV9fHwkSefOnSuocgEAAO65Qhfgjh8/rqpVq2rp0qVasmSJJGnChAlKS0uTq6urzbIuLi6SpPT09HteJwAAQEFxKugCbvbrr79q+vTpio+P10MPPSRJeuONN9SxY0c1aNAgW1C78djNze2OtlOypEfeFIy/zcenWEGXAOAe432P+01BjulCFeD2798vd3d3a3iTpMqVK8vd3V1paWlKSUlRWlqaihQpIklKSEiQJJUpU+aOtpOUlKqsLC58KCg+PsWUkJBS0GXgDvCHF3mhML3vGdPIC/k9ph0cLLkedCpUp1DLlCmj1NRU/f7779a2s2fP6vLly3r00UdVpEgR7dq1y9q3c+dOlS5dWhUqVCiIcgEAAApEoQpw9erVU61atTRmzBgdOHBABw4c0JgxY9SkSRM1bNhQPXv2VHh4uHbv3q3t27drzpw5Cg0NLeiyAQAA7qlCdQrVyclJMTExmjlzpgYOHCiLxaKWLVtqwoQJkqSXXnpJ165d08CBA+Xq6qqePXvq+eefL+CqAQAA7i2L8QDeBZc5cAWLOXDm4+NTTJZXLAVdBkzMmGoUqve9j08xycKYxl0w8n9Mm2YOHAAAAG6PAAcAAGAyBDgAAACTIcABAACYDAEOAADAZAhwAAAAJkOAAwAAMBkCHAAAgMkQ4AAAAEyGAAcAAGAyBDgAAACTIcABAACYDAEOAADAZAhwAAAAJkOAAwAAMBkCHAAAgMkQ4AAAAEyGAAcAAGAyBDgAAACTIcABAACYDAEOAADAZAhwAAAAJkOAAwAAMBkCHAAAgMkQ4AAAAEyGAAcAAGAyBDgAAACTIcABAACYDAEOAADAZAhwAAAAJkOAAwAAMBkCHAAAgMk42bPQ999/r88++0y7du3Sb7/9pitXrqh48eIqV66cHn30UbVr104NGjTI71oBAACg2wS4bdu2KSIiQkeOHFGDBg3UvHlzlS9fXm5ubkpJSdG5c+e0Z88evfvuu6pZs6ZeeOEFBQUF3avaAQAAHki5BrixY8fq559/1rPPPqu2bdvK09Mz15UkJyfr008/1euvv641a9Zozpw5+VIsAAAAbhHgAgMDNXv2bFksltuuxNPTU/369VOfPn20atWqPC0QAAAAtnINcE888cQdr8zBwUFPPvnkXRUEAACAW7PrIoabpaena+/evUpMTFTJkiVVr149ubm55UdtAAAAyMEdBbgffvhBw4cP15UrV+Th4aFLly6paNGimjNnjh599NH8qhEAAAA3uaP7wL3++usaNWqUdu/era+//lo7d+7U008/rcmTJ+dXfQAAAPiLXAPcs88+qx9++MGm7fr163J0dLQ+dnR0lKOjo65fv55/FQIAAMBGrqdQ+/fvr6lTp6pMmTIaNWqUatWqpXHjxumFF17Q9OnTVaxYMSUnJ8vFxUX//ve/72XNAAAAD7RcA1ybNm3Upk0bffbZZxozZoyqVq2qkSNH6uuvv9auXbt04cIFeXt7q169eipatOi9rBkAAOCBdtuLGDp37qyOHTvq448/1vDhw1WnTh2NHDmSixYAAAAKyG0vYrhw4YIsFoueeOIJffbZZwoICNBzzz2nsLAwnTlz5l7UCAAAgJvkGuB27typoKAgBQYGKiAgQGvWrJGTk5N69+6t9evXq0aNGurTp4+mTp2qc+fO3cuaAQAAHmi5BrhXXnlFbdq00bp16zR69GhNmTJFqampkiQXFxc9++yzWr9+vcqVK6eePXves4IBAAAedLkGuLNnz6pVq1aqUqWKOnbsqGvXrikpKclmmSJFimjIkCH6/PPP871QAAAA/OmWV6FOnDhRTZo00cGDB1WzZk09/PDDOS7r4eGRbwUCAADAVq4Bbvr06Vq5cqUOHz6s7t27q1evXrJYLPeyNgAAAOQg1wDn7Oys3r1738taAAAAYIdcA9zo0aM1evRoVaxY0e6VnTx5UpGRkYqKisqL2kzL3d1dRYve0cfMPnB8fIoVdAmF2pUrWbp8+XJBlwEAKKRyDXBBQUHq3bu3GjZsqM6dOysoKEhubm7ZlktNTdX27du1evVq/fDDDxo3bly+FmwGRYs6iLPNuBuG4SDyGwAgN7kGuG7duumxxx7T4sWLNXnyZF25ckWVK1dWuXLl5ObmptTUVJ0/f17Hjh1T0aJF1bt3b7322msqUaLEXRWUmZmpyMhIffTRR0pPT1fLli01depUFS9eXCkpKZo2bZo2b96sokWLasCAAXruuefuansAAABmYzEMw7jdQteuXdM333yjHTt26LffflNqaqo8PT1VtmxZNW3aVE2bNpWLi0ueFDRz5kytW7dOc+bMUdGiRRUWFqZ//OMfmjVrlkaOHKnz589r2rRpOnHihMLCwjR9+nQFBwff0TaSklKVlXXbl/23+fgU4wgc7ophSAkJKQVdhpWPTzFZXmFQ4+8zphqFbkzzixp3xcj/Me3gYFHJkjnf6eO2n4UqSa6urmrdurVat26dp4X9VUpKit59913NmzdPjRs3liSNHTtWc+bM0ZkzZxQfH6+1a9eqWrVqqlmzpo4cOaK4uLg7DnAAAABmVqhm2u/cuVPOzs4KCgqytrVs2VKffPKJ9u7dKy8vL1WrVs3aFxAQoAMHDigjI6MgygUAACgQhSrA/frrr6pQoYK+/PJLhYSEKCgoyPoRXufOnVPp0qVtlvfx8VFmZqYSExMLqGIAAIB7z65TqPfK5cuX9dtvv+mtt97SpEmTZBiGpk+frrCwMNWsWVOurq42y9+Yd5eenn5H28ntfDJQmHCrFdxvGNO43xTkmC5UAc7JyUmXL1/Wv//9b1WtWlWS9Oqrr6pXr16qVatWtqB243FOtze5lXtxEQNwtwrdhG/gLjGmcb8pyIsY7vgU6qVLl3T06FGlp6fn+dyz0qVLy8HBQVWqVLG23fg+KytLCQkJNsufP39ezs7O8vb2ztM6AAAACjO7A9zmzZvVvXt3NWnSRF26dNEvv/yiMWPGaOrUqbp+/XqeFFO/fn1lZWXpp59+srb98ssvcnBwUPfu3ZWUlKTjx49b+3bt2qXatWvn2S1MAAAAzMCuALd+/XoNHTpU1atX16xZs5SVlSXpz09rWLNmjRYuXJgnxVSqVEnt27fXxIkTtW/fPu3bt0+vvPKKOnTooAoVKqhVq1YaP368fvrpJ23YsEGxsbEKDQ3Nk20DAACYhV038g0ODlbz5s0VFham69evy8/PT6tWrZKfn5/eeecdLVu2TBs3bsyTgq5cuaKZM2fq888/l2EY6tChgyZOnCh3d3ddvHhRU6ZM0ZYtW+Tp6annn3/+bwU4buSLwo4b+eJ+w418cd8xw418f/31V5t7s92sVq1a2eam3Y2iRYsqPDxc4eHh2fq8vLw0b968PNsWAACAGdl1CrVixYr69ttvc+zbtWuXKlSokKdFAQAAIHd2HYEbOHCgJk+erNTUVDVv3lwWi0W//PKL/vvf/+rNN99UWFhYftcJAACA/7ErwPXo0UOZmZmKjo7W+++/L0maMGGCPD09NWrUKPXq1StfiwQAAMD/sftGvk8//bSeeuopHTt2TBcvXlTx4sVVpUoVOTo65md9AAAA+Au77wO3adMmzZ49W1WrVlXDhg2VlpamQYMG6ZtvvsnP+gAAAPAXdgW41atXa+jQoTp//ry1rVixYvLw8NDgwYMVHx+fbwUCAADAll0BLiYmRiNGjFBERIS1rXLlypo3b54GDx6s6OjofCsQAAAAtuwKcL///rsaNmyYY1/jxo3166+/5mlRAAAAyJ1dAa5SpUq5ftLC5s2bVbFixTwtCgAAALmz6yrUQYMGaezYsTp//rxatmypEiVK6MKFC9qyZYu++OILzZw5M7/rBAAAwP/YFeAef/xxOTg46K233tKkSZOs7dWrV9fs2bMVHBycbwUCAADAlt33gevcubM6d+6sa9eu6eLFi/Lw8JC7u3t+1gYAAIAc2B3gJCk5OVlpaWnKyspScnKykpOTrX3ly5fP8+IAAACQnV0B7ujRo5o4caL27duXrc8wDFksFh08eDDPiwMAAEB2dgW4qVOnKiEhQa+88orKlCkjBwe7P8ABAAAAecyuALdv3z5FRUWpdevW+V0PAAAAbsOuQ2llypRRZmZmftcCAAAAO9gV4IYNG6bo6GidOHEin8sBAADA7dh1CvXDDz/U2bNn1alTJ3l4eMjV1TXbMtu2bcvz4gAAAJCdXQEuMDBQgYGB+V0LAAAA7GBXgBsxYkR+1wEAAAA72X0j35MnT2r37t3KyMiQYRiSpKysLKWlpWnv3r2aN29evhUJAACA/2P3HLipU6cqKytLFotFkqwhzsHBQU2bNs2/CgEAAGDDrqtQY2Nj1apVK33//fcaMGCAevbsqR9++EELFiyQu7u7QkJC8rtOAAAA/I9dAe706dPq3bu3ihcvrnr16un777+Xq6ur2rRpo+HDhysuLi6/6wQAAMD/2BXg3N3drd8/8sgjOnXqlNLS0iRJfn5+OnnyZP5UBwAAgGzsCnCNGjXS0qVLdfnyZVWuXFlubm6Kj4+XJO3du1dFixbN1yIBAADwf+wKcC+++KIOHDigIUOGyNnZWQMGDNDEiRP1+OOPKzIykjlwAAAA95BdV6FWrVpVGzZssH6U1ogRI1SxYkXt3btXzz77rHr06JGfNQIAAOAmdt8HzsPDQ7Vr17Y+7tatm7p165YfNQEAAOAW7ApwV65c0dKlS/Xjjz8qJSUlx2WWLl2ap4UBAAAgZ3YFuIkTJyo+Pl5NmzZVqVKl8rsmAAAA3IJdAW7r1q2aNm2annrqqfyuBwAAALdh11WopUqV4sgbAABAIWFXgBs9erQiIyO1Z88eXbt2Lb9rAgAAwC3YdQq1SpUqSktLU58+fXJd5uDBg3lWFAAAAHJnV4AbP368MjMzNXz4cE6lAgAAFDC7AtyRI0cUHR2tli1b5nc9AAAAuA275sBVq1ZNFy5cyO9aAAAAYAe77wM3btw4XblyRXXr1pW7u3u2ZSpXrpznxQEAACA7uwJcv379JEnh4eGyWCw2fYZhyGKxcBEDAADAPWJXgONjsgAAAAoPuwJcbGysnnvuOTVt2jS/6wEAAMBt2HURw44dO/K7DgAAANjJrgAXEhKiJUuW6NSpUzIMI79rAgAAwC3YdQr18OHD2rdvn9q3by9HR0d5eXllW2bbtm15XRsAAAByYFeACwwMVGBgYH7XAgAAADvYFeBGjBiR33UAAADATnYFOEk6d+6cFi9erF27dik1NVVeXl6qX7++nnnmGZUrVy4/awQAAMBN7LqI4ejRowoJCdHHH3+sSpUqKSgoSBUqVNDq1avVtWtXHTt2LL/rBAAAwP/YdQRu1qxZevjhh7VkyRJ5eHhY21NTUzVgwADNnj1bixYtyrciAQAA8H/svg/ckCFDbMKbJHl4eGjw4MHauXNnvhQHAACA7OwKcB4eHkpPT8+xLz09nXvDAQAA3EN2BbhmzZpp3rx5Onv2rE37b7/9pujoaDVv3jzPC5syZYqeeuop6+OUlBSNGTNGDRs2VIsWLbRkyZI83yYAAIAZ2DUH7qWXXtLTTz+t9u3bq3bt2ipVqpQSExO1f/9++fj4aPz48Xla1HfffacPPvhAdevWtbZNmjRJ58+f1/Lly3XixAmFhYWpdOnSCg4OztNtAwAAFHa5Brhz586pTJkykqSSJUvq448/1qpVq7R7925dunRJpUuX1tixY9WjR49sc+PuRlpamiZPnqwGDRooMzNTknTmzBnFx8dr7dq1qlatmmrWrKkjR44oLi6OAAcAAB44uQa4kJAQLViwQAEBAQoLC9OwYcMUGhqq0NDQfC0oKipKDRs2VPny5bV161ZJ0t69e+Xl5aVq1apZlwsICNCiRYuUkZEhZ2fnfK0JAACgMMk1wKWnp+v7779XuXLl9NFHH6lt27ZydHTMdUXly5e/62L27t2rdevWae3atVq2bJm1/dy5cypdurTNsj4+PsrMzFRiYiI3EgYAAA+UXANcly5dNG/ePEVHR0vK/eO0DMOQxWLRwYMH76qQ9PR0TZw4URMnTpSXl5dNX1pamlxdXW3aXFxcrM+7UyVL5t0pXyC/+PgUK+gSgDzFmMb9piDHdK4BLjw8XD179lRycrIGDRqkCRMmqEqVKvlWyIIFC1SpUiV17tw5W5+bm1u2oHbjsZub2x1vKykpVVlZ+XfrE35JIS8kJKQUdAlWjGnkBcY07jf5PaYdHCy5HnS65VWoN64Cffjhh9W+fft8PVX56aefKiEhQfXr15ckZWRk6Pr166pfv76mTp2qhIQEm+XPnz8vZ2dneXt751tNAAAAhZFdtxE5f/68Dhw4kK8BbtmyZdarTm883rlzp9544w05OTkpKSlJx48fV+XKlSVJu3btUu3ata2nUgEAAB4UdgW4KlWq6MyZM/laSIUKFWwee3p6ysXFRZUqVZIktWrVSuPHj9e0adN06tQpxcbGasaMGflaEwAAQGFkV4Br0aKF5syZow0bNqhy5coqWbJktmVGjx6d58XdbObMmZoyZYp69+4tT09PjRo1Sp06dcrXbQIAABRGFsOODzJt3br1rVdisWjjxo15VlR+uxcXMVgs+bZ6PAAMo/BN+La8wqDG32dMNQrdmOYXNe6Kkf9j+m9fxHDDV199lacFAQAA4O+zK8Dd8Mcff2j37t1KSEhQhw4d9Mcff6hy5cq3vMEvAAAA8pZdAc4wDEVERGjp0qXKyMiQxWJR3bp1NXfuXCUkJGjx4sUqVapUftcKAAAASQ72LLRgwQK99957mjp1qjZv3qwb0+bGjh2rK1euKCIiIl+LBAAAwP+xK8B98MEHGjNmjHr27CkfHx9re61atTRq1Cjrh84DAAAg/9kV4JKTk633Y/srLy8vXb58OU+LAgAAQO7sCnB+fn5asWKFTZvlf5dfr1u3TrVq1cr7ygAAAJAjuy5ieOmll/Tss8+qS5cuatasmSwWi1avXq3IyEht375dixcvzu86AQAA8D92HYGrX7++VqxYoSpVqujTTz+Vg4OD1q5dKycnJy1fvlyNGzfO7zoBAADwP3bfB65mzZp644038rMWAAAA2OGWAe7gwYNasWKFfvvtN1WoUEFPPvmk/vGPf9yr2gAAAJCDXE+hbt26VT179tTnn3+u5ORkffHFF+rZs6dWrlx5L+sDAADAX+Qa4BYuXKjmzZvr66+/1ooVK7RlyxZ169ZNkZGR97I+AAAA/EWuAe7w4cN65pln5OrqKklydHTUkCFDlJSUpNOnT9+zAgEAAGAr1wB35coVeXh42LSVK1dOkpSSkpK/VQEAACBXuQY4wzCsN+u1Luzw5+JZWVn5WxUAAAByZdd94P7qr8EOAAAA984tbyPyxhtvyMvLy/rYMAxJ0ty5c+Xp6Wmz7Jw5c/K+OgAAAGSTa4Br1KiR0tPTdf78+Wzt165dy9YOAACAeyPXALds2bJ7WQcAAADs9LfmwAEAAKDgEOAAAABMhgAHAABgMgQ4AAAAkyHAAQAAmAwBDgAAwGQIcAAAACZDgAMAADAZAhwAAIDJEOAAAABMhgAHAABgMgQ4AAAAkyHAAQAAmAwBDgAAwGQIcAAAACZDgAMAADAZAhwAAIDJEOAAAABMhgAHAABgMgQ4AAAAkyHAAQAAmAwBDgAAwGQIcAAAACZDgAMAADAZAhwAAIDJEOAAAABMhgAHAABgMgQ4AAAAkyHAAQAAmAwBDgAAwGQIcAAAACZDgAMAADCZQhfgfv/9d40cOVJNmjRRs2bNNHHiRF26dEmSlJKSojFjxqhhw4Zq0aKFlixZUsDVAgAA3HuFKsBlZWVp+PDhunz5suLi4rRo0SIdOnRIYWFhkqRJkybpzJkzWr58uSZNmqR58+Zp3bp1BVw1AADAveVU0AXc7Oeff9b+/fu1bds2+fj4SPoztPXt21dnzpxRfHy81q5dq2rVqqlmzZo6cuSI4uLiFBwcXMCVAwAA3DuF6ghcuXLl9NZbb1nDmyRZLBYZhqGdO3fKy8tL1apVs/YFBATowIEDysjIKIhyAQAACkShCnBeXl4KCgqyaXvnnXdUuXJlJSUlqXTp0jZ9Pj4+yszMVGJi4r0sEwAAoEAVqgD3VzExMfriiy80adIkpaWlydXV1abfxcVFkpSenl4Q5QEAABSIQjUH7mYLFizQvHnzNGXKFLVo0UKHDx/OFtRuPHZzc7ujdZcs6ZFndQL5xcenWEGXAOQpxjTuNwU5pgtlgJsxY4aWLl2qadOmqXfv3pKksmXLKiEhwWa58+fPy9nZWd7e3ne0/qSkVGVlGXlW71/xSwp5ISEhpaBLsGJMIy8wpnG/ye8x7eBgyfWgU6E7hTp//ny9++67mjlzpjW8SZK/v7+SkpJ0/Phxa9uuXbtUu3Zt66lUAACAB0GhCnA///yzFixYoOeff17NmjVTQkKC9ats2bJq1aqVxo8fr59++kkbNmxQbGysQkNDC7psAACAe6pQnUKNj49XVlaWYmJiFBMTY9P32WefaebMmZoyZYp69+4tT09PjRo1Sp06dSqgagEAAAqGxTCM/JsMVkjdizlwFku+rR4PAMMofPOFLK8wqPH3GVONQjem+UWNu2Lk/5g21Rw4AAAA3BoBDgAAwGQIcAAAACZDgAMAADAZAhwAAIDJEOAAAABMhgAHAABgMgQ4AAAAkyHAAQAAmAwBDgAAwGQIcAAAACZDgAMAADAZAhwAAIDJEOAAAABMhgAHAABgMgQ4AAAAkyHAAQAAmAwBDgAAwGQIcAAAACZDgAMAADAZAhwAAIDJEOAAAABMhgAHAABgMgQ4AAAAkyHAAQAAmAwBDgAAwGQIcAAAACZDgAMAADAZAhwAAIDJEOAAAABMhgAHAABgMgQ4AAAAkyHAAQAAmAwBDgAAwGQIcAAAACZDgAMAADAZAhwAAIDJEOAAAABMhgAHAABgMgQ4AAAAkyHAAQAAmAwBDgAAwGQIcAAAACZDgAMAADAZAhwAAIDJEOAAAABMhgAHAABgMgQ4AAAAkyHAAQAAmAwBDgAAwGQIcAAAACZDgAMAADAZAhwAAIDJmC7AZWRkKDw8XE2aNFGTJk0UERGhrKysgi4LAADgnnEq6ALu1Ny5c/XNN98oJiZGly9f1rhx41S8eHENHjy4oEsDAAC4J0x1BO7atWt6//33NXHiRNWrV0+BgYEaM2aMli5dKsMwCro8AACAe8JUAe7gwYNKS0tTQECAtS0gIEAJCQk6ffp0AVYGAABw75jqFOq5c+fk4eEhd3d3a5uPj4+176GHHrJrPQ4Olnyp72aVKuX7JnCfuxfj9E5U8mRQ4+4UtjHNL2rcrfwe07dav6kCXFpamlxdXW3aXFxcJEnp6el2r8fb2/32C92lEyfyfRO4z5Us6VHQJdg4MepEQZcAkytsY5pf1LhbBTmmTXUK1c3NLVtQu/HYzc2tIEoCAAC450wV4MqWLauUlBSlpaVZ2xISEiRJZcqUKaiyAAAA7ilTBbiaNWuqSJEi2rVrl7Vt586dKl26tCpUqFCAlQEAANw7pgpwbm5u6tmzp8LDw7V7925t375dc+bMUWhoaEGXBgAAcM9YDJPdQO3atWuaPn261q1bJ1dXV/Xs2VOjR4+WxVLIrm4CAADIJ6YLcAAAAA86U51CBQAAAAEOAADAdAhwAAAAJkOAQ57p27evhgwZkmPf0aNHVaNGDfn7+ysiIkKnT59WjRo1bvmFB0/r1q1txoCfn58ee+wxzZ49WxkZGXm6rUOHDun777//28+Pjo7WU089lYcV2erfv78iIiKytZ86dUo1atRQfHx8js+LiIhQcHCw9T129OhRRUdH3/K9NmHChHx7Hbg7f31P1KxZUw0bNtRzzz2nI0eO3PX6J0yYoPr16+v333/Pcdvvv/++XetJT0/Xe++9l2PfggUL9OKLL9q0GYah//f//p9atWqlhg0bauDAgTp58uSdv4AHGAEOeSYkJETffPONUlNTs/V99tlnqly5sjZv3qyhQ4eqXLly2rZtm/Wrfv366tevn00bHkxjx461joGNGzfq5Zdf1nvvvaeYmJg83c6wYcN09OjRv/38AQMG5HlN9njooYdUv359bdiwIcf+zz77TF27drW+xx555BENGDDAuk8//PBDSdLy5cutbZMmTbqXLwF36Ob3xNdff60333xTycnJGjZsmLKysu56/VeuXNH06dPvah3r1q3TggULsrWvXr1a8+fPz9b+4YcfaunSpXrttdf04YcfytXVNdcDAMgZAQ55pmPHjpKkr776KlvfjT8qXl5ecnd3l6Ojo3x8fKxfzs7OKlKkiE0bHkweHh7WMVC2bFm1bdtWXbp0yfWIU0Fxd3eXl5dXgWy7S5cu2rRpk65du2bTvnfvXv3222/q0qWL9T3m6Ogod3d36z4tUaKEJMnb29vaVqxYsYJ4GbDTze+J0qVLKyAgQJMmTdLJkyd1+PDhu15/hQoV9MUXX+T4u9tef72hxdWrVzVhwgS98sorqlSpUrblN23apI4dOyowMFBVqlTRyJEjdezYMZ07d+5v1/CgIcAhz3h6eqply5bZjgwcPHhQx48fV5cuXXI9LQTcipOTk5ydnSX9ecpn3Lhx6tGjh5o0aaJdu3YpIyNDs2bNUmBgoAICAvTPf/5Tp06dynV9/fv315kzZzRt2jRNmDBB3333nZo1a6bXX39dDRs21IwZMyRJb7/9ttq1a6fatWurSZMmmjJlivVU7s2nUG88f9WqVWrVqpXq1q2rwYMH648//rBuc8+ePXr66adVt25ddejQQXFxcTZ/9FavXq02bdrI399f06dP1/Xr13Otv3PnzkpPT9fWrVtt2tetW6dGjRqpXLlyNqdQcf+58X5wdHS87fiPj49X586dVadOHbVr107/+c9/bNZVr149de3aVa+++qquXLmS6zY3bdqkLl26qG7duurSpYvWrl0r6c/xHxYWpsTERNWoUUOnT59WUlKSzpw5o5UrV8rf3z/bury9vfX111/r9OnTysjI0OrVq1WuXDnrPxi4PQIc8lRISIi2bt2qy5cvW9vWrVunhg0bqmLFigVYGczo+vXr2r59u9asWaM2bdpY2z/55BM999xzWrx4serUqaPIyEh99913io6O1ooVK+Tj46NnnnlGV69ezXG90dHRKlu2rMaOHWs9fZiYmKizZ8/qo48+Ut++fbVmzRrFxMRo8uTJ2rBhg6ZNm6aPP/5Y69evz3GdFy9e1MqVKzV//nzFxcXpxx9/tJ5iTUxM1MCBA9WuXTt9+umnGjdunN566y3rnKH//ve/mjJlioYMGaJVq1bp6tWrNh8Z+Ffe3t5q3ry5TS1ZWVlav369unbtemc7GaZz7tw5RUVFqXr16qpSpcotx39SUpJGjx6tfv36af369Ro+fLimTZumQ4cO2axzwoQJunLliqKjo3Pc5s8//6wXX3xRoaGhWrt2rZ5//nlNmTJFW7ZsUf369TVx4kSVKFFC27ZtU7ly5VShQgUtW7ZM1atXz3F9L7zwgjw8PNSmTRvVq1dPq1at0sKFC63BFLfnVNAF4P7y2GOPydXVVZs3b1ZwcLCkP0+fMrcB9poxY4b+/e9/S/rzk1ccHR3VpUsXPf/889Zlqlevrscff1zSn6dqli1bpuXLl6tu3bqSpPDwcD322GPasGFDjoHGy8tLjo6O8vDwsDl9OHjwYD388MOSpLNnz+r1119XUFCQpD9PMy1dujTXieOZmZmaOHGi/Pz8JP35z8yPP/4o6c/5ZjcmaktSpUqVlJiYqLffflt9+/bVf/7zH3Xo0EFPPvmkJGnq1KnasmXLLfdTSEiIJk+erPT0dLm4uGjHjh1KTk62TmXA/ePm98T169dlsVgUGBiomJgYZWRk3HL8V69eXRkZGSpTpowqVKigChUqqGzZsipTpozNNkqUKKGxY8dq2rRp6tq1q2rWrGnTHxsbq+7du6tnz56SpIcffljHjh3TkiVL1LJlSxUrVkwODg52T385c+aMHB0dFRUVpQoVKmjx4sUaOXKkVq5cWWBTE8yGAIc85eLioo4dO2r9+vUKDg7W3r17lZCQwB8V2G3IkCHWcObi4qJSpUpl+6/85qO5v/76q9LT0/XMM8/YfKTe1atXdfz4cX3yySeaOnWqtf2f//xnrv9QPPTQQ9bvmzZtqh9//FGRkZE6duyYDh8+rJMnT6phw4a51l65cmXr9x4eHsrMzJT051XY33zzjerXr2/tv379ujIyMpSenq6jR4+qe/fu1j5nZ2fVqlUr1+1IUps2bTR58mRt27ZNrVu31rp169SmTRt5eHjc8nkwnxvvibS0NMXGxmrPnj0aPXq0ypcvr8OHD99y/IeEhKh169YaNmyYKlasqFatWql79+7y9vbOtp2ePXvqo48+0pQpU7KdZj1y5IgOHz6sjz/+2NqWmZn5t055GoahcePGacSIEerUqZMkafbs2Wrfvr1WrVpl888ackeAQ54LCQnRwIEDdeXKFa1bt06tWrVS8eLFC7osmESJEiVynPR8Mzc3N+v3N+aKLV26VJ6enjbLFStWTC4uLqpXr5617a/L5LbeVatWKTw8XD179tRjjz2mF154QdOmTbtlXX8NmjfmuGVmZqpTp0564YUXsj3HyclJFosl2yRwJ6db/3p2dXVV+/bttWHDBgUFBSk+Pl6vv/76LZ8Dc7r5PTFr1iyFhobqn//8p9asWXPb8W+xWLRo0SIdOHBAmzZt0ldffaX3339f8+fPV6tWrWyWt1gseuWVV9S9e/dsAe769esKDQ3NduscB4c7n4n1xx9/6MyZMzb/pDg7O6tGjRq3nLsKW8yBQ54LCAhQyZIltXXrVn3xxRcKCQkp6JJwH3v44Yfl5OSkpKQkVapUSZUqVVL58uU1Z84c/fzzz/Lw8LC2V6pUye7TM++8844GDx6syZMnq0ePHqpSpYp+/fXXbEHLHlWrVtWxY8ds6ti/f7/eeustOTg4yNfXV/v27bMuf/369WxzlHISEhKiTZs2afv27bJYLGrRosUd1wZzcXBw0IwZM5SUlKSIiIjbjv+jR4/qtddek5+fn0aMGKHVq1eradOmuV7VXb16dQ0YMEBz5861uSVU1apVderUKZsxvHnzZq1cuVKSbI7+3Y6Xl5dcXFz0yy+/WNuysrJ0/Phx6xQG3B4BDnnOYrHo8ccf1//7f/9PaWlpatmyZUGXhPuYu7u7evfurVdffVVbt27ViRMn9PLLL+vbb79V1apVb/m8Y8eO6eLFizn2ly5dWt9++62OHj2qQ4cO6aWXXlJCQoLS09PvuMa+ffvq2LFjmjFjho4dO6YtW7YoPDzcehqrf//++uqrr7R06VIdP35cr7/+us6ePXvb9TZp0kRFihTR3LlzFRwcfNujdrg/PPTQQxo8eLA++OADnTx58pbj39PTUx9++KGioqJ06tQpff/99zp06JB1rmZOhg0bJk9PTyUnJ1vbBgwYoC+//FIxMTE6efKkPvnkE0VERKhcuXKSpKJFiyo1NVVHjx61Th3IjaOjo55++mnNnj1bW7du1fHjxxUeHq7k5GSbqQS4NQIc8kVISIh++uknderUiauKkO/GjRuntm3bavz48erWrZtOnz6t2NhYlS5dOtfn9O3bVytXrtTEiRNz7J80aZIyMzP1xBNPaNCgQfLw8FDfvn114MCBO66vbNmyevvtt/XDDz+oa9eumjx5snr16qVRo0ZJkurXr6+5c+dq+fLl6tq1qy5evGjXPz4ODg4KDg7WTz/9xJHuB8ygQYNUsWJFhYeH33L8lypVSgsWLNDXX3+txx9/XKNHj1aPHj3Up0+fXNft5uZmM29UkmrXrq033nhDn376qYKDgxUVFaUxY8aoV69ekv6cM1qtWjV169ZNP/30023rHzdunJ544glNnTpVPXv21MmTJ/Xuu+/mODcPObMYf+d8AAAAAAoMR+AAAABMhgAHAABgMgQ4AAAAkyHAAQAAmAwBDgAAwGQIcAAAACZDgANw32vdurVq1Kih6OjoHPvXrl2rGjVqZPuYoDtx+vRp1ahRQ19//fUd1RUREfG3twngwUWAA/BAsFgs+vLLL3Psy+1jhQCgsCLAAXgg+Pv769ChQzp9+rRN+9WrV7V161b5+voWUGUAcOcIcAAeCA0aNFDJkiWzHYXbunWrvL29bT4bMj09XfPnz1f79u1Vt25dde/eXZs2bbJ53pEjRxQaGip/f38FBwdr//792ba5Z88e9e7dW3Xr1lWLFi0UHR2trKysXGuMiYlRmzZtVLt2bXXo0EHvvvvuXb5qAPcrAhyAB4KDg4Nat26tjRs32rTHx8erQ4cONm1jx47V4sWL1a9fP82fP1/VqlXT0KFDrSEuJSVFoaGhunbtmiIjI/X0009r0qRJNus4dOiQQkND5eXlpejoaA0aNEixsbGaPXt2jvV9/PHHio6O1sCBA7V48WJ17NhRr776qrZs2ZKHewHA/cKpoAsAgHulXbt2Gjp0qC5cuCBvb29lZGRo06ZNeuutt7RixQpJfwavDRs2aPbs2dYPiA8KCtL58+cVFRWlVq1a6aOPPlJqaqoWLlyoEiVKSJIyMzM1a9Ys67YWLVqkhx56SPPnz5ejo6MkqUiRInrllVc0cOBAlSxZ0qa23bt3q0KFCurVq5csFosaN24sZ2dnFSlS5F7sGgAmwxE4AA+MRx99VEWKFLEeSdu+fbuKFCkif39/6zK7du2SxWJRx44dbZ7buXNnHTp0SKmpqdq9e7dq165tDW/Sn+HwZjt27FCzZs1kGIYyMzOVmZmpFi1aKCMjQ7t3785WW0BAgI4fP66ePXvq7bff1vHjxzVixAg1btw4D/cAgPsFAQ7AA8PFxUUtW7a0zoOLj49Xu3btZLFYrMskJyerWLFicnFxsXnujSNmly9f1qVLl+Tt7Z1j/w0XL15UXFyc/Pz8rF8tW7aUJJ0/fz5bbSEhIZoxY4YMw9Ds2bPVsWNH9erVSydOnLjr1w3g/sMpVAAPlHbt2mnChAm6fPmyvvrqK82dO9em39PTUykpKUpPT7cJcYmJidZ+T09P/f777zbPS05OtnlcrFgxPf744+rWrVu2GsqXL59jbT169FCPHj107tw5bdy4UfPmzdOrr76q2NjYv/NSAdzHOAIH4IESFBQkwzC0cOFCZWVlqVGjRjb9DRo0kGEYWr9+vU37559/rlq1asnNzU2NGjXSjz/+aBPitm7darN8/fr1deLECdWpU8f65eTkpKioKCUlJWWra9q0aRo5cqQkqUyZMurTp486dOiQLSgCgMQROAAPGHd3dwUGBiouLk7dunWzXmBwQ61atdS2bVtNmzZNFy9eVOXKlbV27Vp99913mj9/viSpe/fuio2N1eDBgzVy5EglJiZm+5SHIUOGqG/fvgoLC1Pnzp2VnJysyMhIFSlSRJUrV85WV6NGjTR69GhFRUXp0Ucf1a+//qp169apb9+++bczAJgWAQ7AA6ddu3batGlTtgsPbpgzZ46ioqIUExOjS5cuydfXV4sWLVKrVq0k/Xk1aVxcnMLDwzV27Fj5+Pho2rRpGjFihHUd/v7+io2NVVRUlIYPHy53d3c1b95cL730kpydnbNtMzg4WH/88YfeffddxcbGytvbW3369NELL7yQPzsBgKlZDMMwCroIAAAA2I85cAAAACZDgAMAADAZAhwAAIDJEOAAAABMhgAHAABgMgQ4AAAAkyHAAQAAmAwBDgAAwGQIcAAAACbz/wHkUE5JbhEhYgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 720x432 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "\n",
    "# Data\n",
    "models = ['ViT', 'Pre-trained ViT', 'ResNet18']\n",
    "values = [62.17, 96.1, 89.34]\n",
    "\n",
    "# Create the bar chart\n",
    "plt.figure(figsize=(10, 6))\n",
    "plt.bar(models, values, color=['blue', 'green', 'red'])\n",
    "\n",
    "# Add titles and labels\n",
    "plt.title('Model Performance Comparison')\n",
    "plt.xlabel('Models')\n",
    "plt.ylabel('Performance (%)')\n",
    "\n",
    "# Show the plot\n",
    "plt.show()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "gpuType": "T4",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.12"
  },
  "widgets": {
   "application/vnd.jupyter.widget-state+json": {
    "01f313aa78dd4fb9a43eeef2ac9c2f25": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "0dffb96249394778817764ab7ae73f55": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "3905d7327c6c4dcfae873cb34c41603f": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "3d5f89633df44e9f9423de94c8dcb11b": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "4410ab23cd7b47a39e56d43d18aecb29": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "461aba97107a4352899aeebae8c518ec": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "4e4e74e14d414e249fb27163de2b7178": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": ""
     }
    },
    "57cafc338c5c414f828ec78345ade606": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "7508183ccb7e4e3980621ee2beff9d44": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "8c34bc9b82a4426095f9c919d290139f": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": ""
     }
    },
    "98ae0a865b9948c1b6e614d543ce9210": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "9a320ed20f334618badc57ea72809b00": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_bb12c0581b564445bab0f91e2d9a098f",
       "IPY_MODEL_ace7475e6fc840c2b762911fda84ff28",
       "IPY_MODEL_ce8f90eb3ad24e99a3b6668718ebd0a7"
      ],
      "layout": "IPY_MODEL_f5202206586e4c329bbcc39de1c94264"
     }
    },
    "a39e07d1a9ec4fb1a83b73b03d3c3721": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "FloatProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "FloatProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "success",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_0dffb96249394778817764ab7ae73f55",
      "max": 345579424,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_4e4e74e14d414e249fb27163de2b7178",
      "value": 345579424
     }
    },
    "ace7475e6fc840c2b762911fda84ff28": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "FloatProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "FloatProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "success",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_98ae0a865b9948c1b6e614d543ce9210",
      "max": 502,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_8c34bc9b82a4426095f9c919d290139f",
      "value": 502
     }
    },
    "aeddbd0347e84be29068b10d98cfa113": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "bb12c0581b564445bab0f91e2d9a098f": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_cf616281a03046628d95aae643608d96",
      "placeholder": "​",
      "style": "IPY_MODEL_461aba97107a4352899aeebae8c518ec",
      "value": "config.json: 100%"
     }
    },
    "bd53b0fff93e4e77a3a3248e32c77b7a": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_aeddbd0347e84be29068b10d98cfa113",
      "placeholder": "​",
      "style": "IPY_MODEL_3905d7327c6c4dcfae873cb34c41603f",
      "value": " 346M/346M [00:03&lt;00:00, 102MB/s]"
     }
    },
    "ce8f90eb3ad24e99a3b6668718ebd0a7": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_57cafc338c5c414f828ec78345ade606",
      "placeholder": "​",
      "style": "IPY_MODEL_7508183ccb7e4e3980621ee2beff9d44",
      "value": " 502/502 [00:00&lt;00:00, 10.6kB/s]"
     }
    },
    "cf616281a03046628d95aae643608d96": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "d0260e4e3ec145d3a94f9ddd8df87f00": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_01f313aa78dd4fb9a43eeef2ac9c2f25",
      "placeholder": "​",
      "style": "IPY_MODEL_4410ab23cd7b47a39e56d43d18aecb29",
      "value": "model.safetensors: 100%"
     }
    },
    "dfedc00f4ce749f0b3d8048dbae94659": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_d0260e4e3ec145d3a94f9ddd8df87f00",
       "IPY_MODEL_a39e07d1a9ec4fb1a83b73b03d3c3721",
       "IPY_MODEL_bd53b0fff93e4e77a3a3248e32c77b7a"
      ],
      "layout": "IPY_MODEL_3d5f89633df44e9f9423de94c8dcb11b"
     }
    },
    "f5202206586e4c329bbcc39de1c94264": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    }
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}