Lesson_2.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "56bb3b2c-faeb-41ec-8499-508a8bc71b76",
   "metadata": {},
   "source": [
    "# L2: Role of the Tokenizers"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ac6cbd87-6442-4179-9ce9-b096cc37d137",
   "metadata": {},
   "source": [
    "<p style=\"background-color:#fff6e4; padding:15px; border-width:3px; border-color:#f5ecda; border-style:solid; border-radius:6px\"> ⏳ <b>Note <code>(Kernel Starting)</code>:</b> This notebook takes about 30 seconds to be ready to use. You may start and watch the video while you wait.</p>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "6eb3dcf2-4003-414c-a3c4-1e155098be80",
   "metadata": {
    "height": 47
   },
   "outputs": [],
   "source": [
    "import warnings\n",
    "warnings.filterwarnings('ignore')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "e79d2302-3983-4e72-a6b0-262d63fd57ad",
   "metadata": {
    "height": 64
   },
   "outputs": [],
   "source": [
    "training_data = [\n",
    "    \"walker walked a long walk\",\n",
    "]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "837b197c-ec6a-4b74-8cb8-46428c704e55",
   "metadata": {},
   "source": [
    "## BPE - Byte-Pair Encoding"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "8d5379f6-be97-4dcf-98a3-060d88d4d3c6",
   "metadata": {
    "height": 178
   },
   "outputs": [],
   "source": [
    "from tokenizers.trainers import BpeTrainer\n",
    "from tokenizers.models import BPE\n",
    "from tokenizers import Tokenizer\n",
    "from tokenizers.pre_tokenizers import Whitespace\n",
    "\n",
    "bpe_tokenizer = Tokenizer(BPE())\n",
    "bpe_tokenizer.pre_tokenizer = Whitespace()\n",
    "\n",
    "bpe_trainer = BpeTrainer(vocab_size=14)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9d2ab7e1-85dc-4e87-a412-ebe11e78c2ad",
   "metadata": {},
   "source": [
    "<p style=\"background-color:#fff6ff; padding:15px; border-width:3px; border-color:#efe6ef; border-style:solid; border-radius:6px\"> 💻 &nbsp; <b>Access <code>requirements.txt</code> and <code>helper.py</code> files:</b> 1) click on the <em>\"File\"</em> option on the top menu of the notebook and then 2) click on <em>\"Open\"</em>. For more help, please see the <em>\"Appendix - Tips and Help\"</em> Lesson.</p>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "ea4289b9-1761-4dce-8faf-b4be05dab0b2",
   "metadata": {
    "height": 42
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "bpe_tokenizer.train_from_iterator(training_data, bpe_trainer)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "9fd44f86-2aab-46f1-99ec-ea73eac92c7d",
   "metadata": {
    "height": 30
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'al': 10,\n",
       " 'l': 5,\n",
       " 'walk': 12,\n",
       " 'a': 0,\n",
       " 'walke': 13,\n",
       " 'e': 2,\n",
       " 'g': 3,\n",
       " 'w': 9,\n",
       " 'r': 8,\n",
       " 'k': 4,\n",
       " 'd': 1,\n",
       " 'wal': 11,\n",
       " 'n': 6,\n",
       " 'o': 7}"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "bpe_tokenizer.get_vocab()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "74912810-c4ef-4373-86d6-77a095ef01ec",
   "metadata": {
    "height": 42
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['walke', 'r', 'walke', 'd', 'a', 'l', 'o', 'n', 'g', 'walk']"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "bpe_tokenizer.encode(\"walker walked a long walk\").tokens"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "671646d9-3403-4ef7-ae20-72af7fbdad96",
   "metadata": {
    "height": 30
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[9, 5, 4]"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "bpe_tokenizer.encode(\"wlk\").ids"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "22aced61-6e22-4ebe-bd35-60509e744aa2",
   "metadata": {
    "height": 30
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['w', 'l', 'k']"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "bpe_tokenizer.encode(\"wlk\").tokens"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "1e02ebb5-e9e2-4613-948a-655775249ea6",
   "metadata": {
    "height": 30
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['e', 'walke', 'd']"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "bpe_tokenizer.encode(\"she walked\").tokens"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7a1fc751-1074-4c56-a68f-ae6e6491c227",
   "metadata": {},
   "source": [
    "## WordPiece"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "0bbdde63-90d6-493c-8316-96869c366a99",
   "metadata": {
    "height": 178
   },
   "outputs": [],
   "source": [
    "from real_wordpiece.trainer import RealWordPieceTrainer\n",
    "from tokenizers.models import WordPiece\n",
    "\n",
    "real_wordpiece_tokenizer = Tokenizer(WordPiece())\n",
    "real_wordpiece_tokenizer.pre_tokenizer = Whitespace()\n",
    "\n",
    "real_wordpiece_trainer = RealWordPieceTrainer(\n",
    "    vocab_size=27,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "daef2453-f14d-4d77-9f33-b9285bd71920",
   "metadata": {
    "height": 93
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'a': 5,\n",
       " '##ed': 23,\n",
       " 'o': 16,\n",
       " 'r': 14,\n",
       " '##a': 1,\n",
       " '##e': 4,\n",
       " '##lk': 25,\n",
       " '##d': 8,\n",
       " '##g': 11,\n",
       " 'lo': 19,\n",
       " 'walk': 26,\n",
       " '##k': 3,\n",
       " 'd': 15,\n",
       " 'n': 17,\n",
       " '##n': 10,\n",
       " '##er': 22,\n",
       " 'l': 6,\n",
       " '##r': 7,\n",
       " '##o': 9,\n",
       " 'e': 13,\n",
       " '##ng': 20,\n",
       " 'wa': 24,\n",
       " 'g': 18,\n",
       " 'w': 0,\n",
       " '##l': 2,\n",
       " 'k': 12,\n",
       " 'long': 21}"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "real_wordpiece_trainer.train_tokenizer(\n",
    "    training_data, real_wordpiece_tokenizer\n",
    ")\n",
    "real_wordpiece_tokenizer.get_vocab()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "29cc85bd-53c6-41b6-8fba-062ba9f82661",
   "metadata": {
    "height": 42
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['walk', '##er', 'walk', '##ed', 'a', 'long', 'walk']"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "real_wordpiece_tokenizer.encode(\"walker walked a long walk\").tokens"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "455e1f41-e3a1-4366-bfac-44e6ac191aba",
   "metadata": {
    "height": 42
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['w', '##lk']"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "real_wordpiece_tokenizer.encode(\"wlk\").tokens"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "15b04960-d9ba-4037-aeab-fe02f1af6d74",
   "metadata": {},
   "source": [
    "**Unknown Characters:**\n",
    "The following line will produce an error because it contains unknown characters. Please uncomment the line and run it to see the error."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "0bc73f36-ce2c-45d6-b2e6-834e7b7fe24d",
   "metadata": {
    "height": 42
   },
   "outputs": [],
   "source": [
    "#real_wordpiece_tokenizer.encode(\"she walked\").tokens"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "14d662e3-29fc-456c-9b75-1a35fdeaf330",
   "metadata": {},
   "source": [
    "## HuggingFace WordPiece and special tokens"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "b6c9f6ec-7612-4fc0-9de5-fe9af772e8b4",
   "metadata": {
    "height": 212
   },
   "outputs": [],
   "source": [
    "from tokenizers.trainers import WordPieceTrainer\n",
    "\n",
    "unk_token = \"[UNK]\"\n",
    "\n",
    "wordpiece_model = WordPiece(unk_token=unk_token)\n",
    "wordpiece_tokenizer = Tokenizer(wordpiece_model)\n",
    "wordpiece_tokenizer.pre_tokenizer = Whitespace()\n",
    "wordpiece_trainer = WordPieceTrainer(\n",
    "    vocab_size=28,\n",
    "    special_tokens=[unk_token]\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "5e99b0fe-8723-4193-aec4-7af50f4d296d",
   "metadata": {
    "height": 98
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'l': 6,\n",
       " 'lo': 24,\n",
       " '##l': 15,\n",
       " 'wa': 20,\n",
       " 'd': 2,\n",
       " '##r': 19,\n",
       " '##a': 14,\n",
       " '##o': 11,\n",
       " 'n': 7,\n",
       " 'r': 9,\n",
       " '##n': 12,\n",
       " '##lk': 21,\n",
       " 'walk': 22,\n",
       " 'walke': 23,\n",
       " '##ng': 25,\n",
       " 'walker': 27,\n",
       " 'o': 8,\n",
       " '[UNK]': 0,\n",
       " 'g': 4,\n",
       " 'w': 10,\n",
       " '##k': 16,\n",
       " 'k': 5,\n",
       " '##e': 17,\n",
       " 'walked': 26,\n",
       " '##g': 13,\n",
       " 'e': 3,\n",
       " '##d': 18,\n",
       " 'a': 1}"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "wordpiece_tokenizer.train_from_iterator(\n",
    "    training_data, \n",
    "    wordpiece_trainer\n",
    ")\n",
    "wordpiece_tokenizer.get_vocab()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "594722c8-24b0-496b-9859-943071bc7b78",
   "metadata": {
    "height": 42
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['walker', 'walked', 'a', 'lo', '##ng', 'walk']"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "wordpiece_tokenizer.encode(\"walker walked a long walk\").tokens"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "0abca87e-db68-498e-b137-d00a77f034b3",
   "metadata": {
    "height": 30
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['w', '##lk']"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "wordpiece_tokenizer.encode(\"wlk\").tokens"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "ccd22755-1115-4d9a-8ddb-3f3b2128941c",
   "metadata": {
    "height": 42
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['[UNK]', 'walked']"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "wordpiece_tokenizer.encode(\"she walked\").tokens"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d1863790-b1f7-40a3-9fcd-cc4263530f7b",
   "metadata": {},
   "source": [
    "## Unigram"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "963521e9-bab7-4021-b21a-4775f38c994b",
   "metadata": {
    "height": 246
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'o': 7,\n",
       " 'walke': 1,\n",
       " 'w': 10,\n",
       " 'g': 11,\n",
       " 'r': 12,\n",
       " 'a': 5,\n",
       " 'e': 2,\n",
       " 'n': 9,\n",
       " 'k': 3,\n",
       " 'walk': 4,\n",
       " 'd': 8,\n",
       " '[UNK]': 0,\n",
       " 'l': 6}"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from tokenizers.trainers import UnigramTrainer\n",
    "from tokenizers.models import Unigram\n",
    "\n",
    "unigram_tokenizer = Tokenizer(Unigram())\n",
    "unigram_tokenizer.pre_tokenizer = Whitespace()\n",
    "unigram_trainer = UnigramTrainer(\n",
    "    vocab_size=14, \n",
    "    special_tokens=[unk_token],\n",
    "    unk_token=unk_token,\n",
    ")\n",
    "\n",
    "unigram_tokenizer.train_from_iterator(training_data, unigram_trainer)\n",
    "unigram_tokenizer.get_vocab()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "7500af15-5de5-498b-b098-6d1554783257",
   "metadata": {
    "height": 42
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['walke', 'r', 'walke', 'd', 'a', 'l', 'o', 'n', 'g', 'walk']"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "unigram_tokenizer.encode(\"walker walked a long walk\").tokens"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "93ffe74e-968f-4b37-ab73-5686c18a8af7",
   "metadata": {
    "height": 30
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['w', 'l', 'k']"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "unigram_tokenizer.encode(\"wlk\").tokens"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "ee843700-b778-4af3-901c-53d420b12f51",
   "metadata": {
    "height": 42
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['sh', 'e', 'walke', 'd']"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "unigram_tokenizer.encode(\"she walked\").tokens"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "720a62bb-7713-4e88-acb1-621e1da38203",
   "metadata": {
    "height": 42
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[0, 2, 1, 8]"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "unigram_tokenizer.encode(\"she walked\").ids"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e9a3e6cc-4787-4922-a0f9-e98ac5eac5b9",
   "metadata": {
    "height": 30
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7db97e61-b26b-4628-bebd-94290afe3d45",
   "metadata": {
    "height": 30
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "687670ec-9c81-4305-bb7e-377de2d50afa",
   "metadata": {
    "height": 30
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b3cfba8a-6d99-4d12-b786-2a0a5fc9f0d4",
   "metadata": {
    "height": 30
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "942a92bc-ff62-40de-8564-9ae78fee93e7",
   "metadata": {
    "height": 30
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5abef761-87c5-4402-9cd8-092f6a4e1e2a",
   "metadata": {
    "height": 30
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a0b8b6fc-ccac-4e52-96a7-a3fe610b62c8",
   "metadata": {
    "height": 30
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}