tokenization.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "75d7400c-f0c5-4720-8a51-c89e46aefa33",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting nltk\n",
      "  Using cached nltk-3.8.1-py3-none-any.whl.metadata (2.8 kB)\n",
      "Collecting click (from nltk)\n",
      "  Using cached click-8.1.7-py3-none-any.whl.metadata (3.0 kB)\n",
      "Requirement already satisfied: joblib in c:\\coding\\ai\\.venv\\lib\\site-packages (from nltk) (1.4.2)\n",
      "Collecting regex>=2021.8.3 (from nltk)\n",
      "  Downloading regex-2024.5.15-cp311-cp311-win_amd64.whl.metadata (41 kB)\n",
      "     ---------------------------------------- 0.0/42.0 kB ? eta -:--:--\n",
      "     --------- ------------------------------ 10.2/42.0 kB ? eta -:--:--\n",
      "     ------------------ ------------------- 20.5/42.0 kB 320.0 kB/s eta 0:00:01\n",
      "     -------------------------------------  41.0/42.0 kB 326.8 kB/s eta 0:00:01\n",
      "     -------------------------------------  41.0/42.0 kB 326.8 kB/s eta 0:00:01\n",
      "     -------------------------------------- 42.0/42.0 kB 184.5 kB/s eta 0:00:00\n",
      "Collecting tqdm (from nltk)\n",
      "  Downloading tqdm-4.66.4-py3-none-any.whl.metadata (57 kB)\n",
      "     ---------------------------------------- 0.0/57.6 kB ? eta -:--:--\n",
      "     --------------------- ------------------ 30.7/57.6 kB 1.3 MB/s eta 0:00:01\n",
      "     ---------------------------------------- 57.6/57.6 kB 1.0 MB/s eta 0:00:00\n",
      "Requirement already satisfied: colorama in c:\\coding\\ai\\.venv\\lib\\site-packages (from click->nltk) (0.4.6)\n",
      "Using cached nltk-3.8.1-py3-none-any.whl (1.5 MB)\n",
      "Downloading regex-2024.5.15-cp311-cp311-win_amd64.whl (268 kB)\n",
      "   ---------------------------------------- 0.0/269.0 kB ? eta -:--:--\n",
      "   --------------- ------------------------ 102.4/269.0 kB 3.0 MB/s eta 0:00:01\n",
      "   ---------------------------------------- 269.0/269.0 kB 3.3 MB/s eta 0:00:00\n",
      "Using cached click-8.1.7-py3-none-any.whl (97 kB)\n",
      "Downloading tqdm-4.66.4-py3-none-any.whl (78 kB)\n",
      "   ---------------------------------------- 0.0/78.3 kB ? eta -:--:--\n",
      "   ---------------------------------------- 78.3/78.3 kB 4.3 MB/s eta 0:00:00\n",
      "Installing collected packages: tqdm, regex, click, nltk\n",
      "Successfully installed click-8.1.7 nltk-3.8.1 regex-2024.5.15 tqdm-4.66.4\n"
     ]
    }
   ],
   "source": [
    "!pip install nltk"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "c5df283d-2b84-4993-bc57-c76990b5339c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tokens: ['Tokenization', 'is', 'an', 'important', 'NLP', 'technique', '.', 'It', 'involves', 'breaking', 'down', 'text', 'into', 'smaller', 'units', ',', 'such', 'as', 'words', 'or', 'subwords', '.']\n",
      "\n",
      "Stemmed words: ['token', 'is', 'an', 'import', 'nlp', 'techniqu', '.', 'it', 'involv', 'break', 'down', 'text', 'into', 'smaller', 'unit', ',', 'such', 'as', 'word', 'or', 'subword', '.']\n",
      "\n",
      "Lemmatized words: ['Tokenization', 'be', 'an', 'important', 'NLP', 'technique', '.', 'It', 'involve', 'break', 'down', 'text', 'into', 'smaller', 'units', ',', 'such', 'as', 'word', 'or', 'subwords', '.']\n",
      "\n"
     ]
    }
   ],
   "source": [
    "import nltk\n",
    "from nltk.tokenize import word_tokenize\n",
    "from nltk.stem import PorterStemmer, WordNetLemmatizer\n",
    "from nltk.corpus import wordnet\n",
    "\n",
    "\n",
    "# Download necessary NLTK resources (only required once)\n",
    "# nltk.download('punkt')\n",
    "# nltk.download('wordnet')\n",
    "\n",
    "# Sample text\n",
    "text = \"Tokenization is an important NLP technique. It involves breaking down text into smaller units, such as words or subwords.\"\n",
    "\n",
    "\n",
    "# Tokenization\n",
    "tokens = word_tokenize(text)\n",
    "print(\"Tokens:\", tokens)\n",
    "print()\n",
    "\n",
    "# Stemming\n",
    "stemmer = PorterStemmer()\n",
    "stemmed_words = [stemmer.stem(token) for token in tokens]\n",
    "print(\"Stemmed words:\", stemmed_words)\n",
    "print()\n",
    "\n",
    "\n",
    "# Lemmatization\n",
    "lemmatizer = WordNetLemmatizer()\n",
    "lemmatized_words = [lemmatizer.lemmatize(token, wordnet.VERB) for token in tokens]\n",
    "print(\"Lemmatized words:\", lemmatized_words)\n",
    "print()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "980a01ff-cb16-4a7d-a38d-e157f1121e15",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}