main2.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import json\n",
    "from dotenv import load_dotenv\n",
    "from langchain_chroma import Chroma\n",
    "import google.generativeai as genai\n",
    "from langchain.llms import GooglePalm\n",
    "from langchain.chains import RetrievalQA\n",
    "from langchain.prompts import PromptTemplate\n",
    "from langchain.document_loaders import JSONLoader\n",
    "from langchain_google_genai import GoogleGenerativeAI\n",
    "from langchain_huggingface import HuggingFaceEmbeddings\n",
    "from langchain.text_splitter import RecursiveCharacterTextSplitter\n",
    "\n",
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\")\n",
    "\n",
    "# Load environment variables and configure Google Generative AI\n",
    "load_dotenv()\n",
    "genai.configure(api_key=os.environ[\"GOOGLE_API_KEY\"])\n",
    "\n",
    "# Initialize the Google Generative AI model\n",
    "# llm = GoogleGenerativeAI(model=\"models/text-bison-001\", temperature=0.1)\n",
    "llm = GooglePalm(model_name=\"models/text-bison-001\", temperature=0.2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of documents in the loaded vector store: 47\n"
     ]
    }
   ],
   "source": [
    "from langchain.vectorstores import Chroma\n",
    "from langchain.embeddings import HuggingFaceEmbeddings\n",
    "\n",
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\")\n",
    "\n",
    "# Load the embeddings\n",
    "embeddings = HuggingFaceEmbeddings(model_name=\"all-MiniLM-L6-v2\")\n",
    "\n",
    "# Now try to load the vector store\n",
    "loaded_vectorstore = Chroma(persist_directory=\"./chroma_db\", embedding_function=embeddings)\n",
    "print(f\"Number of documents in the loaded vector store: {loaded_vectorstore._collection.count()}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# from mdpdf import markdown_to_pdf\n",
    "\n",
    "# Updated prompt template to specify Markdown output\n",
    "prompt_template = \"\"\"You are an AI assistant tasked with writing a comprehensive document in Markdown format based on a provided table of contents. \n",
    "Use the following pieces of context to write detailed sections for the document.\n",
    "If you don't have enough information, state that more research is needed on that topic.\n",
    "\n",
    "Context: {context}\n",
    "\n",
    "Section to write:\n",
    "{question}\n",
    "\n",
    "AI Assistant: Write a detailed section for the given header in Markdown format. \n",
    "Provide comprehensive and informative content directly related to the section title.\n",
    "Do not create additional headers or a table of contents.\n",
    "Ensure the content is well-structured, relevant to the topic, and flows logically.\"\"\"\n",
    "\n",
    "PROMPT = PromptTemplate(\n",
    "    template=prompt_template, input_variables=[\"context\", \"question\"]\n",
    ")\n",
    "\n",
    "# Update the qa chain with the new prompt\n",
    "qa = RetrievalQA.from_chain_type(\n",
    "    llm=llm,\n",
    "    chain_type=\"stuff\",\n",
    "    retriever=loaded_vectorstore.as_retriever(),\n",
    "    return_source_documents=True,\n",
    "    chain_type_kwargs={\"prompt\": PROMPT}\n",
    ")\n",
    "\n",
    "def notegen(query):\n",
    "    result = qa.invoke({\"query\": query})\n",
    "    return result\n",
    "\n",
    "# answer2 = notegen(query = \"##1. Fundamentals of LLM\")\n",
    "# answer3 = notegen(query = \"1.1 Definition and Key Concepts\")\n",
    "# answer1 = notegen(query = \"\"\"#Understanding Large Language Model Architectures\"\"\")\n",
    "\n",
    "# Convert Markdown content to PDF\n",
    "# pdf_filename = \"output_document.pdf\"\n",
    "# markdown_to_pdf(markdown_content, pdf_filename)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "E0729 07:41:27.147885000 4720809472 client_channel.cc:625]             chand=0x7fb079ccccf0: Illegal keepalive throttling value \n",
      "E0729 07:41:27.147921000 4720809472 client_channel.cc:625]             chand=0x7fb079ccccf0: Illegal keepalive throttling value \n",
      "E0729 07:41:27.147927000 4720809472 client_channel.cc:625]             chand=0x7fb079ccccf0: Illegal keepalive throttling value \n",
      "E0729 07:41:27.147931000 4720809472 client_channel.cc:625]             chand=0x7fb079ccccf0: Illegal keepalive throttling value \n",
      "E0729 07:41:27.147936000 4720809472 client_channel.cc:625]             chand=0x7fb079ccccf0: Illegal keepalive throttling value \n",
      "E0729 07:41:27.147940000 4720809472 client_channel.cc:625]             chand=0x7fb079ccccf0: Illegal keepalive throttling value \n",
      "E0729 07:41:27.147943000 4720809472 client_channel.cc:625]             chand=0x7fb079ccccf0: Illegal keepalive throttling value \n",
      "E0729 07:41:27.147947000 4720809472 client_channel.cc:625]             chand=0x7fb079ccccf0: Illegal keepalive throttling value \n",
      "E0729 07:41:27.147953000 4720809472 client_channel.cc:625]             chand=0x7fb079ccccf0: Illegal keepalive throttling value \n",
      "E0729 07:41:27.147957000 4720809472 client_channel.cc:625]             chand=0x7fb079ccccf0: Illegal keepalive throttling value \n",
      "E0729 07:41:27.147960000 4720809472 client_channel.cc:625]             chand=0x7fb079ccccf0: Illegal keepalive throttling value \n",
      "E0729 07:41:27.147964000 4720809472 client_channel.cc:625]             chand=0x7fb079ccccf0: Illegal keepalive throttling value \n",
      "E0729 07:41:27.147968000 4720809472 client_channel.cc:625]             chand=0x7fb079ccccf0: Illegal keepalive throttling value \n",
      "E0729 07:41:27.147973000 4720809472 client_channel.cc:625]             chand=0x7fb079ccccf0: Illegal keepalive throttling value \n",
      "E0729 07:41:27.147977000 4720809472 client_channel.cc:625]             chand=0x7fb079ccccf0: Illegal keepalive throttling value \n",
      "E0729 07:41:27.147980000 4720809472 client_channel.cc:625]             chand=0x7fb079ccccf0: Illegal keepalive throttling value \n",
      "E0729 07:41:27.147985000 4720809472 client_channel.cc:625]             chand=0x7fb079ccccf0: Illegal keepalive throttling value \n",
      "E0729 07:41:27.147988000 4720809472 client_channel.cc:625]             chand=0x7fb079ccccf0: Illegal keepalive throttling value \n",
      "E0729 07:41:27.147992000 4720809472 client_channel.cc:625]             chand=0x7fb079ccccf0: Illegal keepalive throttling value \n",
      "E0729 07:41:27.354029000 4720809472 client_channel.cc:625]             chand=0x7fb079ccccf0: Illegal keepalive throttling value \n"
     ]
    }
   ],
   "source": [
    "answer1 = notegen(query = \"\"\"#Understanding Large Language Model Architectures\"\"\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "answer3 = notegen(query = \"1.1 Definition and Key Concepts\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "ename": "IndexError",
     "evalue": "list index out of range",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mIndexError\u001b[0m                                Traceback (most recent call last)",
      "Cell \u001b[0;32mIn[7], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m answer2 \u001b[38;5;241m=\u001b[39m \u001b[43mnotegen\u001b[49m\u001b[43m(\u001b[49m\u001b[43mquery\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43m \u001b[49m\u001b[38;5;124;43m\"\"\"\u001b[39;49m\u001b[38;5;124;43m#Fundamentals of LLMs\u001b[39;49m\u001b[38;5;124;43m\"\"\"\u001b[39;49m\u001b[43m)\u001b[49m\n",
      "Cell \u001b[0;32mIn[4], line 32\u001b[0m, in \u001b[0;36mnotegen\u001b[0;34m(query)\u001b[0m\n\u001b[1;32m     31\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mnotegen\u001b[39m(query):\n\u001b[0;32m---> 32\u001b[0m     result \u001b[38;5;241m=\u001b[39m \u001b[43mqa\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43minvoke\u001b[49m\u001b[43m(\u001b[49m\u001b[43m{\u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mquery\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[43mquery\u001b[49m\u001b[43m}\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m     33\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m result\n",
      "File \u001b[0;32m~/miniconda3/envs/spambots/lib/python3.11/site-packages/langchain/chains/base.py:166\u001b[0m, in \u001b[0;36mChain.invoke\u001b[0;34m(self, input, config, **kwargs)\u001b[0m\n\u001b[1;32m    164\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mBaseException\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m e:\n\u001b[1;32m    165\u001b[0m     run_manager\u001b[38;5;241m.\u001b[39mon_chain_error(e)\n\u001b[0;32m--> 166\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m e\n\u001b[1;32m    167\u001b[0m run_manager\u001b[38;5;241m.\u001b[39mon_chain_end(outputs)\n\u001b[1;32m    169\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m include_run_info:\n",
      "File \u001b[0;32m~/miniconda3/envs/spambots/lib/python3.11/site-packages/langchain/chains/base.py:156\u001b[0m, in \u001b[0;36mChain.invoke\u001b[0;34m(self, input, config, **kwargs)\u001b[0m\n\u001b[1;32m    153\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[1;32m    154\u001b[0m     \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_validate_inputs(inputs)\n\u001b[1;32m    155\u001b[0m     outputs \u001b[38;5;241m=\u001b[39m (\n\u001b[0;32m--> 156\u001b[0m         \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_call\u001b[49m\u001b[43m(\u001b[49m\u001b[43minputs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mrun_manager\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mrun_manager\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    157\u001b[0m         \u001b[38;5;28;01mif\u001b[39;00m new_arg_supported\n\u001b[1;32m    158\u001b[0m         \u001b[38;5;28;01melse\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_call(inputs)\n\u001b[1;32m    159\u001b[0m     )\n\u001b[1;32m    161\u001b[0m     final_outputs: Dict[\u001b[38;5;28mstr\u001b[39m, Any] \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mprep_outputs(\n\u001b[1;32m    162\u001b[0m         inputs, outputs, return_only_outputs\n\u001b[1;32m    163\u001b[0m     )\n\u001b[1;32m    164\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mBaseException\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m e:\n",
      "File \u001b[0;32m~/miniconda3/envs/spambots/lib/python3.11/site-packages/langchain/chains/retrieval_qa/base.py:146\u001b[0m, in \u001b[0;36mBaseRetrievalQA._call\u001b[0;34m(self, inputs, run_manager)\u001b[0m\n\u001b[1;32m    144\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m    145\u001b[0m     docs \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_get_docs(question)  \u001b[38;5;66;03m# type: ignore[call-arg]\u001b[39;00m\n\u001b[0;32m--> 146\u001b[0m answer \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mcombine_documents_chain\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mrun\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m    147\u001b[0m \u001b[43m    \u001b[49m\u001b[43minput_documents\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mdocs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mquestion\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mquestion\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mcallbacks\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43m_run_manager\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mget_child\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    148\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    150\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mreturn_source_documents:\n\u001b[1;32m    151\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m {\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39moutput_key: answer, \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124msource_documents\u001b[39m\u001b[38;5;124m\"\u001b[39m: docs}\n",
      "File \u001b[0;32m~/miniconda3/envs/spambots/lib/python3.11/site-packages/langchain_core/_api/deprecation.py:168\u001b[0m, in \u001b[0;36mdeprecated.<locals>.deprecate.<locals>.warning_emitting_wrapper\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m    166\u001b[0m     warned \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mTrue\u001b[39;00m\n\u001b[1;32m    167\u001b[0m     emit_warning()\n\u001b[0;32m--> 168\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mwrapped\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m~/miniconda3/envs/spambots/lib/python3.11/site-packages/langchain/chains/base.py:605\u001b[0m, in \u001b[0;36mChain.run\u001b[0;34m(self, callbacks, tags, metadata, *args, **kwargs)\u001b[0m\n\u001b[1;32m    600\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m(args[\u001b[38;5;241m0\u001b[39m], callbacks\u001b[38;5;241m=\u001b[39mcallbacks, tags\u001b[38;5;241m=\u001b[39mtags, metadata\u001b[38;5;241m=\u001b[39mmetadata)[\n\u001b[1;32m    601\u001b[0m         _output_key\n\u001b[1;32m    602\u001b[0m     ]\n\u001b[1;32m    604\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m kwargs \u001b[38;5;129;01mand\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m args:\n\u001b[0;32m--> 605\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43mkwargs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mcallbacks\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mcallbacks\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mtags\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mtags\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mmetadata\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mmetadata\u001b[49m\u001b[43m)\u001b[49m[\n\u001b[1;32m    606\u001b[0m         _output_key\n\u001b[1;32m    607\u001b[0m     ]\n\u001b[1;32m    609\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m kwargs \u001b[38;5;129;01mand\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m args:\n\u001b[1;32m    610\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mValueError\u001b[39;00m(\n\u001b[1;32m    611\u001b[0m         \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m`run` supported with either positional arguments or keyword arguments,\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m    612\u001b[0m         \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m but none were provided.\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m    613\u001b[0m     )\n",
      "File \u001b[0;32m~/miniconda3/envs/spambots/lib/python3.11/site-packages/langchain_core/_api/deprecation.py:168\u001b[0m, in \u001b[0;36mdeprecated.<locals>.deprecate.<locals>.warning_emitting_wrapper\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m    166\u001b[0m     warned \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mTrue\u001b[39;00m\n\u001b[1;32m    167\u001b[0m     emit_warning()\n\u001b[0;32m--> 168\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mwrapped\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m~/miniconda3/envs/spambots/lib/python3.11/site-packages/langchain/chains/base.py:383\u001b[0m, in \u001b[0;36mChain.__call__\u001b[0;34m(self, inputs, return_only_outputs, callbacks, tags, metadata, run_name, include_run_info)\u001b[0m\n\u001b[1;32m    351\u001b[0m \u001b[38;5;250m\u001b[39m\u001b[38;5;124;03m\"\"\"Execute the chain.\u001b[39;00m\n\u001b[1;32m    352\u001b[0m \n\u001b[1;32m    353\u001b[0m \u001b[38;5;124;03mArgs:\u001b[39;00m\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m    374\u001b[0m \u001b[38;5;124;03m        `Chain.output_keys`.\u001b[39;00m\n\u001b[1;32m    375\u001b[0m \u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[1;32m    376\u001b[0m config \u001b[38;5;241m=\u001b[39m {\n\u001b[1;32m    377\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mcallbacks\u001b[39m\u001b[38;5;124m\"\u001b[39m: callbacks,\n\u001b[1;32m    378\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mtags\u001b[39m\u001b[38;5;124m\"\u001b[39m: tags,\n\u001b[1;32m    379\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mmetadata\u001b[39m\u001b[38;5;124m\"\u001b[39m: metadata,\n\u001b[1;32m    380\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mrun_name\u001b[39m\u001b[38;5;124m\"\u001b[39m: run_name,\n\u001b[1;32m    381\u001b[0m }\n\u001b[0;32m--> 383\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43minvoke\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m    384\u001b[0m \u001b[43m    \u001b[49m\u001b[43minputs\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    385\u001b[0m \u001b[43m    \u001b[49m\u001b[43mcast\u001b[49m\u001b[43m(\u001b[49m\u001b[43mRunnableConfig\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m{\u001b[49m\u001b[43mk\u001b[49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[43mv\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43;01mfor\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[43mk\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mv\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;129;43;01min\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[43mconfig\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mitems\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43;01mif\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[43mv\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;129;43;01mis\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[38;5;129;43;01mnot\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[38;5;28;43;01mNone\u001b[39;49;00m\u001b[43m}\u001b[49m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    386\u001b[0m \u001b[43m    \u001b[49m\u001b[43mreturn_only_outputs\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mreturn_only_outputs\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    387\u001b[0m \u001b[43m    \u001b[49m\u001b[43minclude_run_info\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43minclude_run_info\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    388\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m~/miniconda3/envs/spambots/lib/python3.11/site-packages/langchain/chains/base.py:166\u001b[0m, in \u001b[0;36mChain.invoke\u001b[0;34m(self, input, config, **kwargs)\u001b[0m\n\u001b[1;32m    164\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mBaseException\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m e:\n\u001b[1;32m    165\u001b[0m     run_manager\u001b[38;5;241m.\u001b[39mon_chain_error(e)\n\u001b[0;32m--> 166\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m e\n\u001b[1;32m    167\u001b[0m run_manager\u001b[38;5;241m.\u001b[39mon_chain_end(outputs)\n\u001b[1;32m    169\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m include_run_info:\n",
      "File \u001b[0;32m~/miniconda3/envs/spambots/lib/python3.11/site-packages/langchain/chains/base.py:156\u001b[0m, in \u001b[0;36mChain.invoke\u001b[0;34m(self, input, config, **kwargs)\u001b[0m\n\u001b[1;32m    153\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[1;32m    154\u001b[0m     \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_validate_inputs(inputs)\n\u001b[1;32m    155\u001b[0m     outputs \u001b[38;5;241m=\u001b[39m (\n\u001b[0;32m--> 156\u001b[0m         \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_call\u001b[49m\u001b[43m(\u001b[49m\u001b[43minputs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mrun_manager\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mrun_manager\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    157\u001b[0m         \u001b[38;5;28;01mif\u001b[39;00m new_arg_supported\n\u001b[1;32m    158\u001b[0m         \u001b[38;5;28;01melse\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_call(inputs)\n\u001b[1;32m    159\u001b[0m     )\n\u001b[1;32m    161\u001b[0m     final_outputs: Dict[\u001b[38;5;28mstr\u001b[39m, Any] \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mprep_outputs(\n\u001b[1;32m    162\u001b[0m         inputs, outputs, return_only_outputs\n\u001b[1;32m    163\u001b[0m     )\n\u001b[1;32m    164\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mBaseException\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m e:\n",
      "File \u001b[0;32m~/miniconda3/envs/spambots/lib/python3.11/site-packages/langchain/chains/combine_documents/base.py:138\u001b[0m, in \u001b[0;36mBaseCombineDocumentsChain._call\u001b[0;34m(self, inputs, run_manager)\u001b[0m\n\u001b[1;32m    136\u001b[0m \u001b[38;5;66;03m# Other keys are assumed to be needed for LLM prediction\u001b[39;00m\n\u001b[1;32m    137\u001b[0m other_keys \u001b[38;5;241m=\u001b[39m {k: v \u001b[38;5;28;01mfor\u001b[39;00m k, v \u001b[38;5;129;01min\u001b[39;00m inputs\u001b[38;5;241m.\u001b[39mitems() \u001b[38;5;28;01mif\u001b[39;00m k \u001b[38;5;241m!=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39minput_key}\n\u001b[0;32m--> 138\u001b[0m output, extra_return_dict \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mcombine_docs\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m    139\u001b[0m \u001b[43m    \u001b[49m\u001b[43mdocs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mcallbacks\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43m_run_manager\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mget_child\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mother_keys\u001b[49m\n\u001b[1;32m    140\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    141\u001b[0m extra_return_dict[\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39moutput_key] \u001b[38;5;241m=\u001b[39m output\n\u001b[1;32m    142\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m extra_return_dict\n",
      "File \u001b[0;32m~/miniconda3/envs/spambots/lib/python3.11/site-packages/langchain/chains/combine_documents/stuff.py:249\u001b[0m, in \u001b[0;36mStuffDocumentsChain.combine_docs\u001b[0;34m(self, docs, callbacks, **kwargs)\u001b[0m\n\u001b[1;32m    247\u001b[0m inputs \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_get_inputs(docs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)\n\u001b[1;32m    248\u001b[0m \u001b[38;5;66;03m# Call predict on the LLM.\u001b[39;00m\n\u001b[0;32m--> 249\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mllm_chain\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mpredict\u001b[49m\u001b[43m(\u001b[49m\u001b[43mcallbacks\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mcallbacks\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43minputs\u001b[49m\u001b[43m)\u001b[49m, {}\n",
      "File \u001b[0;32m~/miniconda3/envs/spambots/lib/python3.11/site-packages/langchain/chains/llm.py:318\u001b[0m, in \u001b[0;36mLLMChain.predict\u001b[0;34m(self, callbacks, **kwargs)\u001b[0m\n\u001b[1;32m    303\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mpredict\u001b[39m(\u001b[38;5;28mself\u001b[39m, callbacks: Callbacks \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs: Any) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m \u001b[38;5;28mstr\u001b[39m:\n\u001b[1;32m    304\u001b[0m \u001b[38;5;250m    \u001b[39m\u001b[38;5;124;03m\"\"\"Format prompt with kwargs and pass to LLM.\u001b[39;00m\n\u001b[1;32m    305\u001b[0m \n\u001b[1;32m    306\u001b[0m \u001b[38;5;124;03m    Args:\u001b[39;00m\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m    316\u001b[0m \u001b[38;5;124;03m            completion = llm.predict(adjective=\"funny\")\u001b[39;00m\n\u001b[1;32m    317\u001b[0m \u001b[38;5;124;03m    \"\"\"\u001b[39;00m\n\u001b[0;32m--> 318\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43mkwargs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mcallbacks\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mcallbacks\u001b[49m\u001b[43m)\u001b[49m[\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39moutput_key]\n",
      "File \u001b[0;32m~/miniconda3/envs/spambots/lib/python3.11/site-packages/langchain_core/_api/deprecation.py:168\u001b[0m, in \u001b[0;36mdeprecated.<locals>.deprecate.<locals>.warning_emitting_wrapper\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m    166\u001b[0m     warned \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mTrue\u001b[39;00m\n\u001b[1;32m    167\u001b[0m     emit_warning()\n\u001b[0;32m--> 168\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mwrapped\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m~/miniconda3/envs/spambots/lib/python3.11/site-packages/langchain/chains/base.py:383\u001b[0m, in \u001b[0;36mChain.__call__\u001b[0;34m(self, inputs, return_only_outputs, callbacks, tags, metadata, run_name, include_run_info)\u001b[0m\n\u001b[1;32m    351\u001b[0m \u001b[38;5;250m\u001b[39m\u001b[38;5;124;03m\"\"\"Execute the chain.\u001b[39;00m\n\u001b[1;32m    352\u001b[0m \n\u001b[1;32m    353\u001b[0m \u001b[38;5;124;03mArgs:\u001b[39;00m\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m    374\u001b[0m \u001b[38;5;124;03m        `Chain.output_keys`.\u001b[39;00m\n\u001b[1;32m    375\u001b[0m \u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[1;32m    376\u001b[0m config \u001b[38;5;241m=\u001b[39m {\n\u001b[1;32m    377\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mcallbacks\u001b[39m\u001b[38;5;124m\"\u001b[39m: callbacks,\n\u001b[1;32m    378\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mtags\u001b[39m\u001b[38;5;124m\"\u001b[39m: tags,\n\u001b[1;32m    379\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mmetadata\u001b[39m\u001b[38;5;124m\"\u001b[39m: metadata,\n\u001b[1;32m    380\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mrun_name\u001b[39m\u001b[38;5;124m\"\u001b[39m: run_name,\n\u001b[1;32m    381\u001b[0m }\n\u001b[0;32m--> 383\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43minvoke\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m    384\u001b[0m \u001b[43m    \u001b[49m\u001b[43minputs\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    385\u001b[0m \u001b[43m    \u001b[49m\u001b[43mcast\u001b[49m\u001b[43m(\u001b[49m\u001b[43mRunnableConfig\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m{\u001b[49m\u001b[43mk\u001b[49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[43mv\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43;01mfor\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[43mk\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mv\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;129;43;01min\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[43mconfig\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mitems\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43;01mif\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[43mv\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;129;43;01mis\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[38;5;129;43;01mnot\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[38;5;28;43;01mNone\u001b[39;49;00m\u001b[43m}\u001b[49m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    386\u001b[0m \u001b[43m    \u001b[49m\u001b[43mreturn_only_outputs\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mreturn_only_outputs\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    387\u001b[0m \u001b[43m    \u001b[49m\u001b[43minclude_run_info\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43minclude_run_info\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    388\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m~/miniconda3/envs/spambots/lib/python3.11/site-packages/langchain/chains/base.py:166\u001b[0m, in \u001b[0;36mChain.invoke\u001b[0;34m(self, input, config, **kwargs)\u001b[0m\n\u001b[1;32m    164\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mBaseException\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m e:\n\u001b[1;32m    165\u001b[0m     run_manager\u001b[38;5;241m.\u001b[39mon_chain_error(e)\n\u001b[0;32m--> 166\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m e\n\u001b[1;32m    167\u001b[0m run_manager\u001b[38;5;241m.\u001b[39mon_chain_end(outputs)\n\u001b[1;32m    169\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m include_run_info:\n",
      "File \u001b[0;32m~/miniconda3/envs/spambots/lib/python3.11/site-packages/langchain/chains/base.py:156\u001b[0m, in \u001b[0;36mChain.invoke\u001b[0;34m(self, input, config, **kwargs)\u001b[0m\n\u001b[1;32m    153\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[1;32m    154\u001b[0m     \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_validate_inputs(inputs)\n\u001b[1;32m    155\u001b[0m     outputs \u001b[38;5;241m=\u001b[39m (\n\u001b[0;32m--> 156\u001b[0m         \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_call\u001b[49m\u001b[43m(\u001b[49m\u001b[43minputs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mrun_manager\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mrun_manager\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    157\u001b[0m         \u001b[38;5;28;01mif\u001b[39;00m new_arg_supported\n\u001b[1;32m    158\u001b[0m         \u001b[38;5;28;01melse\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_call(inputs)\n\u001b[1;32m    159\u001b[0m     )\n\u001b[1;32m    161\u001b[0m     final_outputs: Dict[\u001b[38;5;28mstr\u001b[39m, Any] \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mprep_outputs(\n\u001b[1;32m    162\u001b[0m         inputs, outputs, return_only_outputs\n\u001b[1;32m    163\u001b[0m     )\n\u001b[1;32m    164\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mBaseException\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m e:\n",
      "File \u001b[0;32m~/miniconda3/envs/spambots/lib/python3.11/site-packages/langchain/chains/llm.py:129\u001b[0m, in \u001b[0;36mLLMChain._call\u001b[0;34m(self, inputs, run_manager)\u001b[0m\n\u001b[1;32m    123\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m_call\u001b[39m(\n\u001b[1;32m    124\u001b[0m     \u001b[38;5;28mself\u001b[39m,\n\u001b[1;32m    125\u001b[0m     inputs: Dict[\u001b[38;5;28mstr\u001b[39m, Any],\n\u001b[1;32m    126\u001b[0m     run_manager: Optional[CallbackManagerForChainRun] \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m,\n\u001b[1;32m    127\u001b[0m ) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m Dict[\u001b[38;5;28mstr\u001b[39m, \u001b[38;5;28mstr\u001b[39m]:\n\u001b[1;32m    128\u001b[0m     response \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mgenerate([inputs], run_manager\u001b[38;5;241m=\u001b[39mrun_manager)\n\u001b[0;32m--> 129\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mcreate_outputs\u001b[49m\u001b[43m(\u001b[49m\u001b[43mresponse\u001b[49m\u001b[43m)\u001b[49m[\u001b[38;5;241m0\u001b[39m]\n",
      "File \u001b[0;32m~/miniconda3/envs/spambots/lib/python3.11/site-packages/langchain/chains/llm.py:283\u001b[0m, in \u001b[0;36mLLMChain.create_outputs\u001b[0;34m(self, llm_result)\u001b[0m\n\u001b[1;32m    281\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mcreate_outputs\u001b[39m(\u001b[38;5;28mself\u001b[39m, llm_result: LLMResult) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m List[Dict[\u001b[38;5;28mstr\u001b[39m, Any]]:\n\u001b[1;32m    282\u001b[0m \u001b[38;5;250m    \u001b[39m\u001b[38;5;124;03m\"\"\"Create outputs from response.\"\"\"\u001b[39;00m\n\u001b[0;32m--> 283\u001b[0m     result \u001b[38;5;241m=\u001b[39m \u001b[43m[\u001b[49m\n\u001b[1;32m    284\u001b[0m \u001b[43m        \u001b[49m\u001b[38;5;66;43;03m# Get the text of the top generated string.\u001b[39;49;00m\n\u001b[1;32m    285\u001b[0m \u001b[43m        \u001b[49m\u001b[43m{\u001b[49m\n\u001b[1;32m    286\u001b[0m \u001b[43m            \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43moutput_key\u001b[49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43moutput_parser\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mparse_result\u001b[49m\u001b[43m(\u001b[49m\u001b[43mgeneration\u001b[49m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    287\u001b[0m \u001b[43m            \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mfull_generation\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[43mgeneration\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    288\u001b[0m \u001b[43m        \u001b[49m\u001b[43m}\u001b[49m\n\u001b[1;32m    289\u001b[0m \u001b[43m        \u001b[49m\u001b[38;5;28;43;01mfor\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[43mgeneration\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;129;43;01min\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[43mllm_result\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mgenerations\u001b[49m\n\u001b[1;32m    290\u001b[0m \u001b[43m    \u001b[49m\u001b[43m]\u001b[49m\n\u001b[1;32m    291\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mreturn_final_only:\n\u001b[1;32m    292\u001b[0m         result \u001b[38;5;241m=\u001b[39m [{\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39moutput_key: r[\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39moutput_key]} \u001b[38;5;28;01mfor\u001b[39;00m r \u001b[38;5;129;01min\u001b[39;00m result]\n",
      "File \u001b[0;32m~/miniconda3/envs/spambots/lib/python3.11/site-packages/langchain/chains/llm.py:286\u001b[0m, in \u001b[0;36m<listcomp>\u001b[0;34m(.0)\u001b[0m\n\u001b[1;32m    281\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mcreate_outputs\u001b[39m(\u001b[38;5;28mself\u001b[39m, llm_result: LLMResult) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m List[Dict[\u001b[38;5;28mstr\u001b[39m, Any]]:\n\u001b[1;32m    282\u001b[0m \u001b[38;5;250m    \u001b[39m\u001b[38;5;124;03m\"\"\"Create outputs from response.\"\"\"\u001b[39;00m\n\u001b[1;32m    283\u001b[0m     result \u001b[38;5;241m=\u001b[39m [\n\u001b[1;32m    284\u001b[0m         \u001b[38;5;66;03m# Get the text of the top generated string.\u001b[39;00m\n\u001b[1;32m    285\u001b[0m         {\n\u001b[0;32m--> 286\u001b[0m             \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39moutput_key: \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43moutput_parser\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mparse_result\u001b[49m\u001b[43m(\u001b[49m\u001b[43mgeneration\u001b[49m\u001b[43m)\u001b[49m,\n\u001b[1;32m    287\u001b[0m             \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mfull_generation\u001b[39m\u001b[38;5;124m\"\u001b[39m: generation,\n\u001b[1;32m    288\u001b[0m         }\n\u001b[1;32m    289\u001b[0m         \u001b[38;5;28;01mfor\u001b[39;00m generation \u001b[38;5;129;01min\u001b[39;00m llm_result\u001b[38;5;241m.\u001b[39mgenerations\n\u001b[1;32m    290\u001b[0m     ]\n\u001b[1;32m    291\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mreturn_final_only:\n\u001b[1;32m    292\u001b[0m         result \u001b[38;5;241m=\u001b[39m [{\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39moutput_key: r[\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39moutput_key]} \u001b[38;5;28;01mfor\u001b[39;00m r \u001b[38;5;129;01min\u001b[39;00m result]\n",
      "File \u001b[0;32m~/miniconda3/envs/spambots/lib/python3.11/site-packages/langchain_core/output_parsers/base.py:237\u001b[0m, in \u001b[0;36mBaseOutputParser.parse_result\u001b[0;34m(self, result, partial)\u001b[0m\n\u001b[1;32m    222\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mparse_result\u001b[39m(\u001b[38;5;28mself\u001b[39m, result: List[Generation], \u001b[38;5;241m*\u001b[39m, partial: \u001b[38;5;28mbool\u001b[39m \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mFalse\u001b[39;00m) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m T:\n\u001b[1;32m    223\u001b[0m \u001b[38;5;250m    \u001b[39m\u001b[38;5;124;03m\"\"\"Parse a list of candidate model Generations into a specific format.\u001b[39;00m\n\u001b[1;32m    224\u001b[0m \n\u001b[1;32m    225\u001b[0m \u001b[38;5;124;03m    The return value is parsed from only the first Generation in the result, which\u001b[39;00m\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m    235\u001b[0m \u001b[38;5;124;03m        Structured output.\u001b[39;00m\n\u001b[1;32m    236\u001b[0m \u001b[38;5;124;03m    \"\"\"\u001b[39;00m\n\u001b[0;32m--> 237\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mparse(\u001b[43mresult\u001b[49m\u001b[43m[\u001b[49m\u001b[38;5;241;43m0\u001b[39;49m\u001b[43m]\u001b[49m\u001b[38;5;241m.\u001b[39mtext)\n",
      "\u001b[0;31mIndexError\u001b[0m: list index out of range"
     ]
    },
    {
     "ename": "",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31mThe Kernel crashed while executing code in the current cell or a previous cell. \n",
      "\u001b[1;31mPlease review the code in the cell(s) to identify a possible cause of the failure. \n",
      "\u001b[1;31mClick <a href='https://aka.ms/vscodeJupyterKernelCrash'>here</a> for more info. \n",
      "\u001b[1;31mView Jupyter <a href='command:jupyter.viewOutput'>log</a> for further details."
     ]
    }
   ],
   "source": [
    "answer2 = notegen(query = \"\"\"#Fundamentals of LLMs\"\"\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'query': '#Understanding Large Language Model Architectures',\n",
       " 'result': '## Understanding Large Language Model Architectures\\n\\nLarge language models (LLMs) are deep neural networks that have been trained on massive datasets of text, code, or other structured data. They are able to perform a wide variety of natural language processing (NLP) tasks, such as text generation, summarization, translation, and question answering.\\n\\nLLMs are typically built on top of the transformer architecture, which was first introduced in the paper \"Attention Is All You Need\". Transformers are a type of neural network that is specifically designed for processing sequential data, such as text. They work by attending to different parts of the input sequence and then generating an output based on these attended parts.\\n\\nThe transformer architecture has been shown to be very effective for NLP tasks, and it is now the basis of many of the most successful LLMs. However, transformers are also very computationally expensive to train, and they require a large amount of data to achieve good performance.\\n\\nIn recent years, there have been a number of new LLM architectures that have been proposed in an effort to reduce the computational cost and data requirements of training these models. Some of these architectures include:\\n\\n* **Sparse transformers:** Sparse transformers use a sparse attention mechanism, which reduces the number of computations that need to be performed during training.\\n* **Distillation:** Distillation is a technique that can be used to transfer the knowledge from a large LLM to a smaller LLM. This can reduce the computational cost of training the smaller LLM.\\n* **Transfer learning:** Transfer learning is a technique that can be used to fine-tune a pre-trained LLM on a specific task. This can reduce the amount of data that is required to train the LLM for the task.\\n\\nThese new architectures have made it possible to train LLMs that are much larger and more powerful than was previously possible. This has led to a number of new breakthroughs in NLP, and it is likely that LLMs will continue to play an increasingly important role in the field of artificial intelligence.\\n\\n### Transformer Architecture\\n\\nThe transformer architecture is a type of neural network that is specifically designed for processing sequential data, such as text. Transformers work by attending to different parts of the input sequence and then generating an output based on these attended parts.\\n\\nThe transformer architecture consists of two main components: an encoder and a decoder. The encoder is responsible for encoding the input sequence into a fixed-length vector representation. The decoder is then responsible for decoding this vector representation into the output sequence.\\n\\nThe encoder and decoder are both made up of a stack of identical layers. Each layer in the encoder consists of a self-attention mechanism and a feed-forward network. The self-attention mechanism allows the encoder to attend to different parts of the input sequence and learn the relationships between them. The feed-forward network then transforms the output of the self-attention mechanism into a new vector representation.\\n\\nThe decoder is similar to the encoder, but it also has an additional attention mechanism that allows it to attend to the output of the encoder. This attention mechanism allows the decoder to generate the output sequence in a way that is consistent with the input sequence.\\n\\nThe transformer architecture has been shown to be very effective for NLP tasks, such as text generation, summarization, translation, and question answering. This is because the transformer architecture is able to learn the long-range dependencies that are present in natural language.\\n\\n### Sparse Transformers\\n\\nSparse transformers are a type of transformer architecture that uses a sparse attention mechanism. A sparse attention mechanism reduces the number of computations that need to be performed during training, which can significantly reduce the computational cost of training a transformer model.\\n\\nThe sparse attention mechanism works by only attending to a small number of tokens in the input sequence. This is done by using a mask to filter out the tokens that are not relevant to the current prediction. The mask is learned during training, and it is based on the attention weights that are computed by the transformer model.\\n\\nSparse transformers have been shown to achieve comparable performance to dense transformers, while using significantly less computation. This makes them a good choice for applications where the computational cost of training a transformer model is a concern.\\n\\n### Distillation\\n\\nDistillation is a technique that can be used to transfer the knowledge from a large LLM to a smaller LLM. This can reduce the computational cost of training the smaller LLM.\\n\\nDistillation works by training the smaller LLM to mimic the predictions of the larger LLM. This is done by using a loss function that measures the difference between the predictions of the two models. The smaller LLM is then trained to minimize this loss function.\\n\\nDistillation has been shown to be a very effective technique for transferring knowledge from large LLMs to smaller LLMs. This is because the larger LLMs have typically been trained on a much larger dataset, and they have therefore learned a more general representation',\n",
       " 'source_documents': [Document(metadata={'seq_num': 1, 'source': '/Users/tusharchandra/Workspace/repos/Notegen/LLM_Primer.json'}, page_content='Open-Source Language Models with Longtermism by Bi et al. from the DeepSeek-AI team, DeepSeek LLM is a project aimed at advancing the capabilities of open-source language models with a long-term perspective. Building upon the foundations laid by previous literature which presented varied conclusions on scaling LLMs, this paper presents novel findings that facilitate the scaling of large-scale models in two commonly used open-source configurations: 7B and 67B parameters. At the core of their approach is the development of a dataset comprising 2 trillion tokens, which supports the pre-training phase. Additionally, supervised fine-tuning (SFT) and direct preference optimization (DPO) are conducted on the base models to create the DeepSeek Chat models. Through rigorous evaluation, DeepSeek LLM 67B demonstrates its superiority over LLaMA-2 70B across various benchmarks, especially in code, mathematics, and reasoning, and exhibits enhanced performance in open-ended evaluations against GPT-3.5. The architecture of DeepSeek LLM adheres closely to the LLaMA design, utilizing a Pre-Norm structure with RMSNorm, SwiGLU for the Feed-Forward Network (FFN), and incorporating Rotary Embedding for positional encoding. Modifications include a 30-layer network for the 7B model and a 95-layer network for the 67B model, differing in layer adjustments to optimize training and inference efficiency. A critical contribution of this study is the exploration of scaling laws for hyperparameters, where optimal values for batch size and learning rate are identified based on extensive experimentation. This leads to a significant revelation: the quality of training data critically impacts the optimal scaling strategy between model size and data volume. The higher the data quality, the more a scaling budget should lean towards model scaling. The paper also delves into alignment strategies through SFT and DPO, employing a dataset with 1.5 million instances to enhance the model\\\\u2019s helpfulness and harmlessness. The evaluation framework spans across a wide array of public benchmarks in both English and Chinese, addressing various domains such as language understanding, reasoning, and coding. Safety evaluation forms a pivotal part of the study, ensuring that the models adhere to ethical guidelines and are devoid of harmful outputs. The results across multiple safety evaluation metrics underscore the model\\\\u2019s reliability and safe interaction capabilities. The DeepSeek LLM initiative not only pushes the envelope in the open-source landscape of LLMs but also sets a new benchmark for future research in scaling, safety, and alignment of language models, driving forward the quest towards Artificial General Intelligence (AGI). Liberated-Qwen1.5 Abacus.AI has released an intriguing new open-source large language model called Liberated-Qwen1.5-72B. What makes this LLM unique is its strict adherence to system prompts, even when those prompts conflict with the user\\\\u2019s instructions. As enterprises adopt LLMs for uses like customer service chatbots, getting models to reliably follow set guidelines is crucial. Too often, existing LLMs can veer off in unexpected directions when conversing with users over multiple turns. Liberated-Qwen aims to solve this with its system prompt compliance. The team at Abacus fine-tuned the Alibaba Qwen1.5-72B model on a new 7K SystemChat dataset specifically designed to train obedience to system messages, even when contradicting user input. The result is an uncensored model that will execute directives like answering only in capitals. Initial tests show Liberated-Qwen performing slightly better than the original Qwen model on areas like coding (HumanEval) while maintaining strong general capabilities (MMLU scores). However, it lacks safety guardrails, so caution is advised before deployment. The team plans to further improve the model\\\\u2019s coding performance and release enhanced versions blending the SystemChat data with the training used for their previous Smaug release. This innovative approach could make Liberated-Qwen and its successors compelling options for businesses needing LLMs that prioritize obedience to rules and guidelines, but getting safe deployment right will be key. Command-R Cohere\\\\u2019s Command-R is optimized for long context tasks such as retrieval augmented generation (RAG), using external APIs and tools. It boasts low latency, high accuracy, supports 10 languages, and has 128k context length. The model excels at 10 major languages of global business: English, French, Spanish, Italian, German, Portuguese, Japanese, Korean, Arabic, and Chinese. Hugging Face Command R+ C4AI Command R+ is an open weights research release of a 104B billion parameter model with highly advanced capabilities, this includes Retrieval Augmented Generation (RAG) and tool use to automate sophisticated tasks. The tool use in this model generation enables multi-step tool use which allows the model to combine multiple tools over multiple steps to accomplish difficult tasks. C4AI Command R+ is a multilingual model evaluated in 10 languages for performance: English, French, Spanish, Italian, German, Brazilian Portuguese, Japanese, Korean, Arabic, and Simplified Chinese. Command R+ is optimized for a variety of use cases including reasoning, summarization, and question answering. Hugging Face ; Demo ; EagleX EagleX 1.7T is a early research release of a 7.52B parameter model training that is Built on the RWKV-v5 architecture , a linear transformer with 10-100x+ lower inference cost. Is continuation based on the original Eagle 7B model Ranks as the world\\\\u2019s greenest 7B model (per token) Trained on 1.7 Trillion tokens across 100+ languages Outperforms all 7B class models in multi-lingual benchmarks Passes LLaMA2 (2T) in multiple English evals, approaches Mistral (>2T?) All while being an \\\\u201cAttention-Free Transformer\\\\u201d Hugging Face ; Hugging Face Demo Grok Grok-1 xAI has released the base model weights and network architecture of Grok-1, a 314 billion parameter Mixture-of-Experts model trained from scratch by xAI using a custom training stack on top of JAX and Rust in October 2023. This is the raw base model checkpoint from the Grok-1 pre-training phase, which concluded in October 2023. This means that the model is not fine-tuned for any specific application, such as dialogue. Code Grok-1.5 Grok-1.5 is introduced as the latest advancement in long context understanding and advanced reasoning, promising availability to early testers and existing users on the X platform shortly. Following the release of Grok-1\\\\u2019s model weights and network architecture, xAI\\\\u2019s Grok-1.5 showcases enhanced reasoning and problem-solving capabilities, particularly highlighted in coding and math-related tasks. In performance benchmarks, Grok-1.5 demonstrated significant improvements by achieving a 50.6% score on the MATH benchmark, a 90% score on the GSM8K benchmark, and a 74.1% score on the'),\n",
       "  Document(metadata={'seq_num': 1, 'source': '/Users/tusharchandra/Workspace/repos/Notegen/LLM_Primer.json'}, page_content='on the exact same data, in the exact same order. Pythia allows public access to 154 checkpoints for each model, with tools to download and reconstruct their exact training data, offering insights into memorization, term frequency effects on few-shot performance, and reducing gender bias. The Pythia model suite was deliberately designed to promote scientific research on large language models, especially interpretability research. Despite not centering downstream performance as a design goal, they find the models match or exceed the performance of similar and same-sized models, such as those in the OPT and GPT-Neo suites. The following table from the paper shows commonly used model suites and how they rate according to theirmerg requirements. The suite\\\\u2019s consistent setup across models is used to analyze gender bias mitigation by modifying training data\\\\u2019s gendered terms, demonstrating reduced bias measures in larger models. Another focus is memorization dynamics, where memorization is modeled as a Poisson point process, indicating that memorization occurrences are uniformly distributed throughout training, contrary to the theory that later training data is memorized more. The study also explores the impact of term frequency in pretraining data on model performance, finding a correlation between term frequency and task accuracy in larger models, an emergent property not observed in smaller models. The paper, presented at the International Conference on Machine Learning (ICML) 2023, emphasizes the utility of Pythia for detailed analysis and research on LLM behaviors, offering a new perspective on how pretraining data affects model development. Hugging Face . Orca Orca 13B, a small yet mighty AI model developed by Microsoft that\\\\u2019s making waves in the AI community. Despite its size, Orca 13B is proving that it can stand toe-to-toe with the giants, demonstrating capabilities that rival even the larger foundation models (LFMs) like ChatGPT and GPT-4. Orca 13B\\\\u2019s progressive learning approach is a cornerstone of its success. By learning from rich signals from GPT-4, including explanation traces, step-by-step thought processes, and other complex instructions, Orca is able to develop a deeper understanding of the reasoning process. This is a significant departure from traditional AI models, which often focus on imitating the style of LFMs but fail to capture their reasoning process. The use of explanation traces, for instance, allows Orca to understand the underlying logic behind the responses generated by GPT-4. This not only enhances Orca\\\\u2019s ability to generate accurate responses, but also enables it to understand the context and nuances of different scenarios, thereby improving its overall performance. Furthermore, the role of ChatGPT as a teacher assistant is crucial in providing a supportive learning environment for Orca. By providing guidance and feedback, ChatGPT helps Orca refine its learning process and improve its understanding of complex instructions. This teacher-student dynamic is a key factor in Orca\\\\u2019s ability to imitate the reasoning process of LFMs. Orca\\\\u2019s performance in various benchmarks is a testament to its capabilities. In complex zero-shot reasoning benchmarks like Big-Bench Hard (BBH) and AGIEval, Orca surpasses conventional state-of-the-art instruction-tuned models such as Vicuna-13B by more than 100% and 42% respectively. This is a significant achievement, considering that these benchmarks are designed to test the model\\\\u2019s ability to reason and make decisions in complex scenarios. One of the most remarkable aspects of Orca is its size. Despite being a smaller AI model compared to giants like ChatGPT, Orca manages to perform at the same level. This is a significant breakthrough in technology as it demonstrates that powerful AI models can be built by smaller teams, making AI development more accessible. The size of Orca also has implications for its efficiency and scalability. Being a smaller model, Orca requires less computational resources to train and operate, making it a more sustainable and cost-effective solution for AI development. Furthermore, its smaller size makes it easier to scale and adapt to different applications, thereby increasing its versatility and utility. phi-1 Proposed in Textbooks Are All You Need by Gunasekar from Microsoft Research, phi-1 is a large language model for code, with significantly smaller size than competing models: phi-1 is a Transformer-based model with 1.3B parameters, trained for 4 days on 8 A100s, using a selection of \\\\u201ctextbook quality\\\\u201d data from the web (6B tokens) and synthetically generated textbooks and exercises with GPT-3.5 (1B tokens). Despite this small scale, phi-1 attains pass@1 accuracy 50.6% on HumanEval and 55.5% on MBPP. It also displays surprising emergent properties compared to phi-1-base, the model before their finetuning stage on a dataset of coding exercises, and phi-1-small, a smaller model with 350M parameters trained with the same pipeline as phi-1 that still achieves 45% on HumanEval. They demonstrate that increasing layer count and sacrificing computational cost is not the only approach to increase LLM accuracy, but instead focusing on data quality can also result in a significant accuracy boost \\\\u2013 reinforcing the fact that a data-centric approach also helps in making your model better. XGen A series of 7B LLMs named XGen-7B from Salesforce with standard dense attention on up to 8K sequence length for up to 1.5T tokens. We also fine tune the models on public-domain instructional data. The main take-aways are: On standard NLP benchmarks, XGen achieves comparable or better results when compared with state-of-the-art open-source LLMs (e.g. MPT, Falcon, LLaMA, Redpajama, OpenLLaMA) of similar model size. Our targeted evaluation on long sequence modeling benchmarks show benefits of our 8K-seq models over 2K- and 4K-seq models. XGen-7B archives equally strong results both in text (e.g., MMLU, QA) and code (HumanEval) tasks. Training cost of $150K on 1T tokens under Google Cloud pricing for TPU-v4. Project page ; Code ; Model checkpoint . OpenLLMs OpenLLMs is a series of open-source language models fine-tuned on a small, yet diverse and high-quality dataset of multi-round conversations. Specifically, we utilize only ~6K GPT-4 conversations directly filtered from the ~90K ShareGPT conversations. Despite the small size of the dataset, OpenLLMs has demonstrated remarkable performance. Generic Models: OpenChat : based on LLaMA-13B with a context length of 2048. Achieves 105.7% of ChatGPT score on the Vicuna GPT-4 evaluation. Achieves 80.9% win-rate on AlpacaEval. OpenChat-8192 : based on LLaMA-13B, with an extended context length of 8192. Achieves 106.6% of ChatGPT score on the Vicuna GPT-4 evaluation. Achieves 79.5% win-rate on AlpacaEval. Code Models: OpenCoderPlus : based on StarCoderPlus with a native context length of 8192. Achieves 102.5% of'),\n",
       "  Document(metadata={'seq_num': 1, 'source': '/Users/tusharchandra/Workspace/repos/Notegen/LLM_Primer.json'}, page_content='a detailed example and where you learn how to fine-tune Llama 7B to generate synthetic instructions, e.g., provide an email to get instructions that could have been used to generate this email. This model can then be used to generate synthetic data for personalizing LLMs, e.g., mimic your email writing. How to make LLMs go fast The post presents an extensive overview of various methods to enhance the performance of LLMs, focusing on aspects like improved hardware utilization and innovative decoding techniques. The goal is to offer a valuable starting point for further exploration of the topics of interest, with an effort to provide links to pertinent research papers and blog posts where relevant. Topics covered: How inference works Compilers Continuous Batching Quantization and model shrinkage KV caching Speculative decoding Training time optimization Merging LLMs Merging two LLMs can yield the \\\\u201cbest of both worlds\\\\u201d, i.e., an LLM that is proficient at the best performing areas of the individual LLMs. For instance, is -performing 7B param model on the Open LLM Leaderboard Remarkably, it also ranks as the 10th best-performing model overall. In just 7B parameters! NeuralBeagle14-7B is a DPO fine-tune of mlabonne/Beagle14-7B by Maxime Labonne using the argilla/distilabel-intel-orca-dpo-pairs preference dataset and my DPO notebook from this article . It is based on a merge of the following models using LazyMergekit : fblgit/UNA-TheBeagle-7b-v1 argilla/distilabeled-Marcoro14-7B-slerp Demo (Space) ; Article about merging models ; Article about DPO fine-tuning References How does the dot product determine similarity? Cosine similarity versus dot product as distance metrics Augmented Language Models: a Survey SAI Notes #08: LLM based Chatbots to query your Private Knowledge Base Prompt Engineering Guide Less is More: Why Use Retrieval Instead of Larger Context Windows The Secret Sauce behind 100K context window in LLMs: all tricks in one place Citation If you found our work useful, please cite it as: @article{Chadha2020DistilledLLMs, title = {Overview of Large Language Models}, author = {Chadha, Aman}, journal = {Distilled AI}, year = {2020}, note = {\\\\\\\\url{https://aman.ai}} } | | | | www.amanchadha.com\"}'),\n",
       "  Document(metadata={'seq_num': 1, 'source': '/Users/tusharchandra/Workspace/repos/Notegen/LLM_Primer.json'}, page_content='{\"title\": \"Aman\\'s AI Journal \\\\u2022 Primers \\\\u2022 Overview of Large Language Models\", \"url\": \"https://aman.ai/primers/ai/LLM/\", \"page_content\": \"Aman\\'s AI Journal \\\\u2022 Primers \\\\u2022 Overview of Large Language Models Distilled AI Back to aman.ai Primers \\\\u2022 Overview of Large Language Models Overview Embeddings Contextualized vs. Non-Contextualized Embeddings Use-cases of Embeddings Similarity Search with Embeddings Dot Product Similarity Geometric intuition Cosine Similarity Cosine similarity vs. dot product similarity How do LLMs work? LLM Training Steps Reasoning Retrieval/Knowledge-Augmented Generation or RAG (i.e., Providing LLMs External Knowledge) Process Summary Vector Database Feature Matrix Context Length Extension Challenges with Context Scaling The \\\\u201cNeedle in a Haystack\\\\u201d Test Status quo RAG vs. Ultra Long Context (1M+ Tokens) Solutions to Challenges Positional Interpolation (PI) Rotary Positional Encoding (RoPE) ALiBi (Attention with Linear Biases) Sparse Attention Flash Attention Multi-Query Attention Comparative Analysis Dynamically Scaled RoPE Approach Key Benefits NTK-Aware Method Perspective Summary Related: Traditional DBs v/s Vector DBs When not to use Vector DBs? Knowledge Graphs with LLMs: Best of Both Worlds Continuous v/s Discrete Knowledge Representation The \\\\u201cContext Stuffing\\\\u201d Problem RAG for limiting hallucination LLM Knobs Token Sampling Prompt Engineering Token healing Evaluation Metrics Methods to Knowledge-Augment LLMs Fine-tuning vs. Prompting RAG RAG vs. Fine-tuning Augmenting LLMs with Knowledge Graphs Motivation Process Summary of LLMs Leaderboards Open LLM Leaderboard LMSYS Chatbot Arena Leaderboard Massive Text Embedding Benchmark (MTEB) Leaderboard Big Code Models Leaderboard Open VLM Leaderboard LLM Safety Leaderboard AlpacaEval Leaderboard Hallucination Leaderboard LLM-Perf Leaderboard Vectara\\\\u2019s Hallucination Leaderboard YALL - Yet Another LLM Leaderboard Artificial Analysis Leaderboard Martian\\\\u2019s Provider Leaderboard Enterprise Scenarios Leaderboard Extending prompt context Motivation Status quo Extending Context Length via RoPE scaling Summary of tricks to optimize attention/memory usage for extending prompt context Large prompt context models Scaling Transformer to 1M tokens and beyond with RMT Hyena Hierarchy: Towards Larger Convolutional Language Models LongNet: Scaling Transformers to 1,000,000,000 Tokens Extending Context Window of Large Language Models via Positional Interpolation Recent techniques powering LLMs Popular LLMs Popular Foundation LLMs Llama LLaMA Llama 2 Llama 3 Llama 3.1 GPT GPT-3.5 Turbo GPT-4 GPT-4 Turbo GPT-4o GPT-4o mini Bard API Claude Claude 2.1 Claude 3 Claude 3.5 Alpaca Vicuna StableVicuna Dolly 2.0 StableLM OpenLLaMA MPT Falcon The RefinedWeb Dataset for Falcon LLM RedPajama Pythia Orca phi-1 XGen OpenLLMs LlongMA-2 Qwen Qwen 2 Mistral 7B Mixtral 8x7B MoE Summary Mixtral 8x22B MoE Zephyr: Direct Distillation of LM Alignment Mistral NeMo HuggingFace\\\\u2019s Alignment Handbook Yi Yi 1.5 effi Starling NexusRaven-V2 Llama Guard Notus OpenChat phi-1.5 Phi-2 DeciLM LLM360 OLMo DeepSeek LLM Liberated-Qwen1.5 Command-R Command R+ EagleX Grok Grok-1 Grok-1.5 SaulLM DBRX Jamba WizardLM-2 Gemini Gemma Gemma 2: Improving Open Language Models at a Practical Size JetMoE Popular Medical LLMs Med-PaLM Med-PaLM 1 Med-PaLM 2 MediTron-70B BioMistral OpenBioLLM Popular Indic LLMs OpenHathi BharatGPT Kannada Llama Tamil-LLaMA Ambari Krutrim BengaliGPT Gajendra Airavata MalayaLLM Hanooman Navarasa 2.0 Gujju Llama 1.0 Pragna Popular Code LLMs SQLCoder Panda-Coder Magicoder AlphaCode 2 Phind-70B Granite StarCoder2 CodeGemma DeepSeek-Coder-V2 Frameworks LangChain: Build apps with LLMs Cheatsheet Resources YouTube Videos Deploying LangChain LangChain with MLFlow LangChain with NeMo Guardrails (building safe and secure apps) LangFlow - GUI for LangChain Popular use cases examples LlamaIndex Flowise RAGAS LLaMA2-Accessory LLaMA Factory GPTCache Prompt Lookup Decoding Axolotl TRL - Transformer Reinforcement Learning Miscellaneous Estimate Token Importance in LLM Prompts Attention Manipulation to Steer LLM Output Strategies to get better results using prompt engineering The Reversal Curse: LLMs trained on \\\\u201cA is B\\\\u201d fail to learn \\\\u201cB is A\\\\u201d Further Reading Llama 2: Responsible Use Guide Extended Guide: Instruction-tune Llama 2 How to make LLMs go fast Merging LLMs References Citation Overview Large Language Models (LLMs) are deep neural networks that utilize the Transformer architecture. LLMs are part of a class of models known as foundation models because these models can be transferred to a number of downstream tasks (via fine-tuning) since they have been trained on a huge amount of unsupervised and unstructured data. The Transformer architecture has two parts: encoder and decoder. Both encoder and decoder are mostly identical (with a few differences); (more on this in the primer on the Transformer architecture). Also, for the pros and cons of the encoder and decoder stack, refer Autoregressive vs. Autoencoder Models . Given the prevalence of decoder-based models in the area of generative AI, the article focuses on decoder models (such as GPT-x) rather than encoder models (such as BERT and its variants). Henceforth, the term LLMs is used interchangeably with \\\\u201cdecoder-based models\\\\u201d. \\\\u201cGiven an input text \\\\u201cprompt\\\\u201d, at essence what these systems do is compute a probability distribution over a \\\\u201cvocabulary\\\\u201d\\\\u2014the list of all words (or actually parts of words, or tokens) that the system knows about. The vocabulary is given to the system by the human designers. Note that GPT-3, for example, has a vocabulary of about 50,000 tokens.\\\\u201d Source It\\\\u2019s worthwhile to note that while LLMs still suffer from a myriad of limitations, such as hallucination and issues in chain of thought reasoning (there have been recent improvements), it\\\\u2019s important to keep in mind that LLMs were trained to perform statistical language modeling. Specifically, language modeling is defined as the task of predicting the next token given some context. Embeddings In the context of Natural Language Processing (NLP), embeddings are dense vector representations of words or sentences that capture semantic and syntactic properties of words or sentences. These embeddings are usually obtained by training models, such as BERT and its variants, Word2Vec, GloVe, or FastText, on a large corpus of text, and they provide a way to convert textual information into a form that machine learning algorithms can process. Put simply, embeddings encapsulate the semantic meaning of words (which are internally represented as one or more tokens) or semantic and syntactic properties of sentences by representing them as dense, low-dimensional vectors. Note that embeddings can be contextualized (where the embeddings of each token are a function of other tokens in the input; in particular, this')]}"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "answer1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'answer2' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "Cell \u001b[0;32mIn[27], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m \u001b[43manswer2\u001b[49m\n",
      "\u001b[0;31mNameError\u001b[0m: name 'answer2' is not defined"
     ]
    }
   ],
   "source": [
    "answer2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'query': '1.1 Definition and Key Concepts',\n",
       " 'result': '## 1.1 Definition and Key Concepts\\n\\nA large language model (LLM) is a type of neural network that has been trained on a massive dataset of text, typically in the order of billions of words. This allows LLMs to learn the relationships between words and phrases, and to generate text that is both coherent and fluent. LLMs have been used for a variety of tasks, including natural language processing, machine translation, and code generation.\\n\\n### Key Concepts\\n\\n* **Context:** The context is the text that is provided to the LLM before it generates output. The context can be used to provide the LLM with information about the topic of the output, or to give it a starting point for its generation.\\n* **Prompt:** The prompt is the text that is provided to the LLM after the context. The prompt can be used to provide the LLM with additional instructions or constraints on its output.\\n* **Output:** The output is the text that is generated by the LLM. The output can be a single sentence, a paragraph, or even a long document.\\n\\n### Challenges\\n\\nLLMs are still under development, and there are a number of challenges that need to be addressed before they can be used for a wider range of tasks. These challenges include:\\n\\n* **Hallucination:** LLMs can sometimes generate text that is nonsensical or factually incorrect. This is because LLMs are not able to distinguish between real and imaginary information.\\n* **Bias:** LLMs can sometimes generate text that is biased or offensive. This is because LLMs are trained on data that is often biased or offensive.\\n* **Scalability:** LLMs are very computationally expensive to train and deploy. This makes it difficult to use LLMs for tasks that require real-time responses or that need to be run on resource-constrained devices.\\n\\n### Applications\\n\\nLLMs have been used for a variety of tasks, including:\\n\\n* **Natural language processing:** LLMs can be used for tasks such as text summarization, translation, and question answering.\\n* **Machine translation:** LLMs can be used to translate text from one language to another.\\n* **Code generation:** LLMs can be used to generate code, such as Python or Java.\\n* **Artificial intelligence assistants:** LLMs can be used to create artificial intelligence assistants that can help users with tasks such as scheduling appointments, finding information, and composing emails.\\n\\nLLMs are a powerful tool that has the potential to be used for a wide range of tasks. However, there are still a number of challenges that need to be addressed before LLMs can be used for all tasks.',\n",
       " 'source_documents': [Document(metadata={'seq_num': 1, 'source': '/Users/tusharchandra/Workspace/repos/Notegen/LLM_Primer.json'}, page_content='traditional web data usage. This approach aims to enhance common sense reasoning in natural language and achieves performance comparable to models five times its size. It excels in complex reasoning tasks like grade-school mathematics and basic coding, while showing characteristics of larger models, including the ability to think step by step and some in-context learning capabilities. However, it also shares some negative traits like hallucinations and potential for biased outputs, though improvements are noted in the absence of web data. Phi-1.5 shares the same architecture as phi-1: a Transformer with 24 layers, 32 heads (each head with dimension 64), rotary embedding, context length of 2048, and flash-attention for training speedup. The training data comprises 7 billion tokens from phi-1\\\\u2019s data and 20 billion tokens of new synthetic data focusing on common sense reasoning. Additionally, two variants, phi-1.5-web-only and phi-1.5-web, were created to investigate the value of traditional web data. Phi-1.5-web-only was trained purely on 95 billion tokens of filtered web data, while phi-1.5-web used a mix of filtered web data, phi-1\\\\u2019s code data, and synthetic NLP data. Phi-1.5 was evaluated on standard natural language benchmarks including common sense reasoning, language understanding, mathematics, and coding. It achieved results comparable to models like Llama2-7B, Falcon-7B, and Vicuna-13B in these benchmarks. In reasoning tasks, including elementary school math and entry-level Python coding, phi-1.5 outperformed all existing models, including Llama 65B. The addition of web data in phi-1.5-web showed significant improvements on these reasoning tasks. The plot below from the paper illustrates benchmark results comparing phi-1.5, its version enhanced with filtered web data phi-1.5-web, and other state-of-the-art open-source LLMs. Sizes range from phi-1.5\\\\u2019s 1.3 billion parameters (Falcon-RW-1.3B) to 10x larger models like Vicuna-13B, a fine-tuned version of Llama-13B). Benchmarks are broadly classified into three categories: common sense reasoning, language skills, and multi-step reasoning. The classification is meant to be taken loosely, for example while HellaSwag requires common sense reasoning, it arguably relies more on \\\\u201cmemorized knowledge\\\\u201d. One can see that phi-1.5 models perform comparable in common sense reasoning and language skills, and vastly exceeds other models in multi-step reasoning. Note that the numbers are from our own evaluation pipeline, to ensure consistency between models, and thus they might differ slightly from numbers reported elsewhere. The paper acknowledges the challenge of toxic and biased content generation in language models. To assess this, they used the ToxiGen dataset and crafted 86 prompts to test the model\\\\u2019s responses. Phi-1.5 showed an improvement over models like Llama2-7B and Falcon-7B, passing more prompts and failing fewer compared to these models. Phi-1.5 and phi-1.5-web, despite not undergoing instruction-based finetuning, demonstrated the ability to comprehend and execute basic human instructions and chat capabilities. This ability is attributed to the synthetic textbook data used in training, which included exercises and answers. The paper describes standard prompting techniques and showcases the model\\\\u2019s flexibility in natural language processing and code generation. Phi-2 Microsoft\\\\u2019s Research team has been addressing inefficiencies in LLMs, specifically the trade-off between size and performance. Smaller models traditionally underperform in tasks like coding, common-sense reasoning, and language understanding compared to their larger counterparts. By advancing a suite of Small Language Models (SLMs), named \\\\u201cPhi\\\\u201d, Microsoft aims to bridge this performance gap, ensuring that more compact models can still deliver high levels of accuracy and utility in various applications. Introduced in Phi-2: The surprising power of small language models by Javaheripi and Bubeck. The article details Microsoft Research\\\\u2019s release of Phi-2, a 2.7 billion-parameter language model. This model is part of the \\\\u201cPhi\\\\u201d series of SLMs, including Phi-1 (1.3 billion parameters) and Phi-1.5 (also 1.3 billion parameters). Phi-2 stands out for its exceptional capabilities, achieving equivalent language understanding capabilities to models 5x larger and matching reasoning capabilities of models up to 25x larger. The Phi series of models scale down the number of parameters without a proportional loss in performance. Phi-1 showcased this in coding benchmarks, performing on par with larger models. With Phi-1.5 and the latest Phi-2, Microsoft has implemented novel model scaling techniques and refined training data curation to achieve results comparable to models many times their size. The success of Phi-2, a 2.7 billion-parameter language model, signifies a leap in optimization that allows it to demonstrate state-of-the-art reasoning and language understanding, matching or exceeding models with up to 25 times more parameters. Phi-2\\\\u2019s success relies on two core strategies: firstly, Phi-2\\\\u2019s prowess stems from a relentless focus on high-quality \\\\u201ctextbook-quality\\\\u201d data, integrating synthetic datasets designed to impart common sense reasoning and general knowledge. Thus, highly selected/curated/generated data used in Phi-2\\\\u2019s training to educate the model on some specific foundational capabilities (e.g., common sense reasoning, problem solving, math, etc) is central to Phi-2\\\\u2019s exceptional performance. Secondly, it utilizes innovative scaling techniques by building upon the knowledge embedded in the 1.3 billion parameter Phi-1.5, employing scaled knowledge transfer for enhanced performance and faster training convergence. By valuing textbook-caliber content and embedding knowledge from its predecessor Phi-1.5, Phi-2 emerges as a powerhouse in reasoning and comprehension. This scaled knowledge transfer not only accelerates training convergence but shows clear boost in Phi-2 benchmark scores, as shown in the graphs from the blog below. The model, which is a Transformer-based model, was trained on 1.4 trillion tokens from a mix of Synthetic and Web datasets, over 14 days using 96 A100 GPUs. Notably, Phi-2 has not undergone alignment through reinforcement learning from human feedback (RLHF) or been instruct fine-tuned, yet demonstrates improved behavior regarding toxicity and bias. Phi-2 is so small that it can run on a device, thus opening the door to a bunch of very interesting edge scenarios where latency or data sensitivity (e.g., for personalization) is paramount. Phi-2\\\\u2019s performance is highlighted in several benchmarks, including Big Bench Hard (BBH), commonsense reasoning, language understanding, math, and coding tasks, often surpassing or matching other models like Mistral, Llama-2, and Gemini Nano 2 despite its smaller size. Additionally, the article presents'),\n",
       "  Document(metadata={'seq_num': 1, 'source': '/Users/tusharchandra/Workspace/repos/Notegen/LLM_Primer.json'}, page_content='and hybrid data stores. Backfilling can be very slow if your historical dataset is huge. Knowledge Graphs with LLMs: Best of Both Worlds Credits to the following section go to Tony Seale . The recent increasing significance on LLMs within organisations is not just a fleeting fad but part of a transformative shift that all forward-thinking organisations must come to terms with. However, for an organisation to succeed in this transition, effectively leveraging ontologies (of which knowledge graphs are a popular instantiation) is a crucial factor. LLMs possess remarkable AI capabilities, allowing them to comprehend and generate human-like text by learning intricate patterns from vast volumes of training data. These powerful models are capable of crafting eloquent letters, analysing data, generating code, orchestrating workflows, and performing a myriad of other complex tasks. Their potential seems increasingly disruptive, with Microsoft even \\\\u2018betting the house\\\\u2019 on them. However, when deploying LLMs within an enterprise context, reliability, trustworthiness, and understandability are vital concerns for those running and governing these systems. Hallucination is simply not an option. Ontologies offer structured and formal representations of knowledge, defining relationships between concepts within specific domains. These structures enable computers to comprehend and reason in a logical, consistent, and comprehensible manner. Yet, designing and maintaining ontologies requires substantial effort. Before LLMs came along, they were the \\\\u2018top dog in town\\\\u2019 when it came to a semantic understanding, but now they seem relatively inflexible, incomplete and slow to change. Enter the intriguing and powerful synergy created by the convergence of LLMs AND Ontologies. The ability of LLMs to generate and extend ontologies is a game-changer. Although you still need a \\\\u2018human-in-the-loop,\\\\u2019 the top LLMs demonstrate surprising effectiveness. Simultaneously, ontologies provide vital context to the prompts given to LLMs, enriching the accuracy and relevance of the LLM\\\\u2019s responses. Ontologies can also be used to validate the consistency of those responses. LLMs can help discover new knowledge, and the ontologies compile that knowledge down for future use. This collaborative partnership between LLMs and ontologies establishes a reinforcing feedback loop of continuous improvement. As LLMs help generate better ontologies faster and more dynamically, the ontologies, in turn, elevate the performance of LLMs by offering a more comprehensive context of the data and text they analyse. This positive feedback loop has the potential to catalyse an exponential leap in the capabilities of AI applications within organisations, streamlining processes, adding intelligence, and enhancing customer experiences like never before. Continuous v/s Discrete Knowledge Representation Credits to the following section go to Tony Seale . We can think of information existing in a continuous stream or in discrete chunks. LLMs fall under the category of continuous knowledge representation, while Knowledge Graphs belong to the discrete realm. Each approach has its merits, and understanding the implications of their differences is essential. LLM embeddings are dense, continuous real-valued vectors existing in a high-dimensional space. Think of them as coordinates on a map: just as longitude and latitude can pinpoint a location on a two-dimensional map, embeddings guide us to rough positions in a multi-dimensional \\\\u2018semantic space\\\\u2019 made up of the connections between the words on the internet. Since the embedding vectors are continuous, they allow for an infinite range of values within a given interval, making the embeddings\\\\u2019 coordinates \\\\u2018fuzzy\\\\u2019. An LLM embedding for \\\\u2018Jennifer Aniston\\\\u2019 will be a several-thousand-dimensional continuous vector that leads to a location in a several-billion-parameter \\\\u2018word-space\\\\u2019. If we add the \\\\u2018TV series\\\\u2019 embedding to this vector then I will be pulled towards the position of the \\\\u2018Friends\\\\u2019 vector. Magic! But this magic comes with a price: you can never quite trust the answers. Hallucination and creativity are two sides of the same coin. On the other hand, Knowledge Graphs embrace a discrete representation approach, where each entity is associated with a unique URL. For example, the Wikidata URL for Jennifer Aniston is https://www.wikidata.org/wiki/Q32522 . This represents a discrete location in \\\\u2018DNS + IP space\\\\u2019. Humans have carefully structured data that is reliable, editable, and explainable. However, the discrete nature of Knowledge Graphs also comes with its own price. There is no magical internal animation here; just static facts. The \\\\u201cContext Stuffing\\\\u201d Problem Research shows that providing LLMs with large context windows \\\\u2013 \\\\u201ccontext stuffing\\\\u201d \\\\u2013 comes at a cost and performs worse than expected. Less is More: Why Use Retrieval Instead of Larger Context Windows summarizes two studies showing that: LLMs tend to struggle in distinguishing valuable information when flooded with large amounts of unfiltered information. Put simply, answer quality decreases, and the risk of hallucination increases, with larger context windows. Using a retrieval system to find and provide narrow, relevant information boosts the models\\\\u2019 efficiency per token, which results in lower resource consumption and improved accuracy. The above holds true even when a single large document is put into the context, rather than many documents. Costs increase linearly with larger contexts since processing larger contexts requires more computation. LLM providers charge per token which means a longer context (i.e, more tokens) makes each query more expensive. LLMs seem to provide better results when given fewer, more relevant documents in the context, rather than large numbers of unfiltered documents. RAG for limiting hallucination Hallucination is typically caused due to imperfections in training data, lack of access to external, real-world knowledge, and limited contextual understanding from prompts. RAG (using either an agent or an external data-source such as a Vector DB) can serve as a means to alleviate model hallucination and improve accuracy. Furthermore, augmenting the prompt using examples is another effective strategy to reduce hallucination. Another approach which has recently gained traction is plan-and-execute where the model is asked to first plan and then solve the problem step-by-step while paying attention to calculations. Lastly, as contaminated training data can cause hallucinations, cleaning up the data and fine-tuning your model can also help reduce hallucinations. However, as most models are large to train or even fine-tune, this approach should be used while taking the cost-vs-accuracy tradeoff into consideration. LLM Knobs When working with prompts, you interact with the LLM'),\n",
       "  Document(metadata={'seq_num': 1, 'source': '/Users/tusharchandra/Workspace/repos/Notegen/LLM_Primer.json'}, page_content='architecture, we have successfully increased the model\\\\u2019s effective context length to an unprecedented two million tokens, while maintaining high memory retrieval accuracy. Their method allows for the storage and processing of both local and global information and enables information flow between segments of the input sequence through the use of recurrence. - Their experiments demonstrate the effectiveness of RMT, which holds significant potential to enhance long-term dependency handling in natural language understanding and generation tasks as well as enable large-scale context processing for memory-intensive applications. The following figure from the paper shows memory-intensive synthetic tasks. Synthetic tasks and the required RMT operations to solve them are presented. In the Memorize task, a fact statement is placed at the start of the sequence. In the Detect and Memorize task, a fact is randomly placed within a text sequence, making its detection more challenging. In the Reasoning task, two facts required to provide an answer are randomly placed within the text. For all tasks, the question is at the end of the sequence. \\\\u2019mem\\\\u2019 denotes memory tokens, \\\\u2019Q\\\\u2019 represents the question, and \\\\u2019A\\\\u2019 signifies the answer. Hyena Hierarchy: Towards Larger Convolutional Language Models Recent advances in deep learning have relied heavily on the use of large Transformers due to their ability to learn at scale. However, the core building block of Transformers, the attention operator, exhibits quadratic cost in sequence length, limiting the amount of context accessible. Existing subquadratic methods based on low-rank and sparse approximations need to be combined with dense attention layers to match Transformers, indicating a gap in capability. This paper by Poli et al. from proposes Hyena, a subquadratic drop-in replacement for attention constructed by interleaving implicitly parametrized long convolutions and data-controlled gating. Guided by these findings, we introduce the Hyena hierarchy, an operator defined by a recurrence of two efficient subquadratic primitives: a long convolution and element-wise multiplicative gating (see figure below from the paper). A specified depth (i.e., number of steps) of the recurrence controls the size of the operator. For short recurrences, existing models are recovered as special cases. By mapping each step in the Hyena recurrence to its corresponding matrix form, we reveal Hyena operators to be equivalently defined as a decomposition of a data-controlled matrix i.e., a matrix whose entries are functions of the input. Furthermore, we show how Hyena operators can be evaluated efficiently without materializing the full matrix, by leveraging fast convolution algorithms. Empirically, Hyena operators are able to significantly shrink the quality gap with attention at scale, reaching similar perplexity and downstream performance with a smaller computational budget and without hybridization of attention. The following figure from the paper illustrates the Hyena operator is defined as a recurrence of two efficient subquadratic primitives: an implicit long convolution \\\\\\\\(h\\\\\\\\) (i.e. Hyena filters parameterized by a feed-forward network) and multiplicative elementwise gating of the (projected) input. The depth of the recurrence specifies the size of the operator. Hyena can equivalently be expressed as a multiplication with data-controlled (conditioned by the input \\\\\\\\(u\\\\\\\\)) diagonal matrices \\\\\\\\(\\\\\\\\mathrm{D}_x\\\\\\\\) and Toeplitz matrices \\\\\\\\(\\\\\\\\mathrm{S}_h\\\\\\\\). In addition, Hyena exhibits sublinear parameter scaling (in sequence length) and unrestricted context, similar to attention, while having lower time complexity. In recall and reasoning tasks on sequences of thousands to hundreds of thousands of tokens, Hyena improves accuracy by more than 50 points over operators relying on state-spaces and other implicit and explicit methods, matching attention-based models. We set a new state-of-the-art for dense-attention-free architectures on language modeling in standard datasets (WikiText103 and The Pile), reaching Transformer quality with a 20% reduction in training compute required at sequence length 2K. Hyena operators are twice as fast as highly optimized attention at sequence length 8K, and 100x faster at sequence length 64K. LongNet: Scaling Transformers to 1,000,000,000 Tokens Scaling sequence length has become a critical demand in the era of large language models. However, existing methods struggle with either computational complexity or model expressivity, rendering the maximum sequence length restricted. This paper by Ding et al. from Furu Wei\\\\u2019s group at MSR introduces LongNet, a Transformer variant that can scale sequence length to more than 1 billion tokens, without sacrificing the performance on shorter sequences. Specifically, they propose dilated attention, which expands the attentive field exponentially as the distance grows. LongNet has significant advantages: It has a linear computation complexity and a logarithm dependency between tokens; It can be served as a distributed trainer for extremely long sequences; Its dilated attention is a drop-in replacement for standard attention, which can be seamlessly integrated with the existing Transformer-based optimization. The following figure from the paper illustrates the trend of Transformer sequence lengths over time. Experiments results demonstrate that LongNet yields strong performance on both long-sequence modeling and general language tasks. Our work opens up new possibilities for modeling very long sequences, e.g., treating a whole corpus or even the entire Internet as a sequence. Code Extending Context Window of Large Language Models via Positional Interpolation This paper by Chen et al. from Meta AI in 2023 presents Position Interpolation (PI) that extends the context window sizes of RoPE-based pretrained LLMs such as LLaMA models to up to 32768 with minimal fine-tuning (within 1000 steps), while demonstrating strong empirical results on various tasks that require long context, including passkey retrieval, language modeling, and long document summarization from LLaMA 7B to 65B. Meanwhile, the extended model by Position Interpolation preserve quality relatively well on tasks within its original context window. To achieve this goal, Position Interpolation linearly down-scales the input position indices to match the original context window size, rather than extrapolating beyond the trained context length which may lead to catastrophically high attention scores that completely ruin the self-attention mechanism. They present a theoretical study which shows that the upper bound of interpolation is at least \\\\u223c600x smaller than that of extrapolation, further demonstrating its stability. Models extended via Position Interpolation retain its original architecture and can reuse most pre-existing optimization and infrastructure. The following figure from the paper'),\n",
       "  Document(metadata={'seq_num': 1, 'source': '/Users/tusharchandra/Workspace/repos/Notegen/LLM_Primer.json'}, page_content='tasks. In performance benchmarks, Grok-1.5 demonstrated significant improvements by achieving a 50.6% score on the MATH benchmark, a 90% score on the GSM8K benchmark, and a 74.1% score on the HumanEval benchmark, showcasing its prowess in a wide range of mathematical and coding challenges. A notable feature of Grok-1.5 is its ability to process contexts up to 128,000 tokens, expanding its memory capacity significantly to handle information from longer documents. This is visualized through a graph indicating a 100% recall rate across varying context lengths, emphasizing the model\\\\u2019s robust information retrieval capacity even with extensive contexts. Grok-1.5\\\\u2019s infrastructure is based on a custom distributed training framework integrating JAX, Rust, and Kubernetes, designed for the demanding requirements of LLM research. This framework addresses challenges in training LLMs on large compute clusters by optimizing reliability, uptime, and efficient resource management through a custom training orchestrator and improvements in checkpointing, data loading, and training job restarts. SaulLM Proposed in SaulLM-7B: A pioneering Large Language Model for Law , SaulLM-7B, introduced by Colombo et al., is the first LLM with 7 billion parameters, designed specifically for the legal domain, based on the Mistral 7B architecture. It is trained on over 30 billion tokens from an English legal corpus, showing state-of-the-art proficiency in legal document comprehension and generation. Additionally, the paper introduces an instructional fine-tuning method using legal datasets to enhance SaulLM-7B\\\\u2019s performance on legal tasks, released under the MIT License. The creation of SaulLM-7B addresses the gap in specialized LLM applications within the legal field, marked by complex document volumes and unique linguistic challenges. The model\\\\u2019s pretraining incorporates extensive legal corpora from English-speaking jurisdictions, including the USA, Canada, the UK, and Europe, aiming to comprehend and adapt to the evolving legal discourse. This focus targets the needs of legal practitioners, representing a significant step towards integrating artificial intelligence within legal applications. SaulLM-7B\\\\u2019s family includes SaulLM-7B-Instruct, an instruction-tuned variant that outperforms models like Mistral and Llama on various legal tasks. This achievement is part of the paper\\\\u2019s contributions, which also introduce LegalBench-Instruct and model evaluation code & licensing under the MIT License. LegalBench-Instruct, a refined iteration of LegalBench, aims to better assess and refine legal language model proficiency, incorporating tasks from the MMLU benchmark related to international law, professional law, and jurisprudence. The paper details the data sources and preprocessing steps involved in constructing the training corpus, highlighting the combination of pre-existing datasets and new data scraped from the web. Rigorous data cleaning, deduplication, and the inclusion of \\\\u201creplay\\\\u201d sources to mitigate catastrophic forgetting during continued pretraining form the foundation of a robust 30 billion token corpus. Instruction fine-tuning mixes further refine the model\\\\u2019s ability to understand and follow legal instructions. Evaluation of SaulLM-7B involves comparing its performance against other open-source models using benchmarks like LegalBench-Instruct and Legal-MMLU. The results demonstrate SaulLM-7B\\\\u2019s superior proficiency in legal document processing and task performance. The perplexity analysis across different types of legal documents further confirms the model\\\\u2019s effectiveness in the legal domain. SaulLM-7B signifies a novel approach in the AI-driven assistance for legal professionals, aiming for widespread adoption and innovation across commercial and research endeavors in law. The release of SaulLM-7B under an open license encourages collaborative development and application in various legal contexts, setting a precedent for future advancements in AI-powered legal tools. The model is open-sourced and allows commercial use, inviting the legal sector and AI engineers to further tinker with legal LLMs. Model DBRX DBRX is a new state-of-the-art Open LLM by Databricks Mosaic Research Team that achieves state-of-the-art performance across multiple benchmarks, surpassing existing open models like GPT-3.5 and showing competitive results against Gemini 1.0 Pro. DBRX is notable for its exceptional capabilities in coding tasks, even outperforming specialized models such as CodeLLaMA-70B. Key features and advancements of DBRX include: Efficiency and Performance : DBRX introduces significant efficiency improvements in training and inference through its fine-grained mixture-of-experts (MoE) architecture. It boasts inference speeds up to 2x faster than LLaMA2-70B and has about 40% fewer parameters compared to Grok-1, without compromising on model quality. The model demonstrates a quadruple reduction in computational requirements compared to its predecessors while maintaining similar performance levels. Architecture Innovations : The model utilizes a transformer-based decoder-only structure with 132B total parameters, incorporating advanced techniques such as rotary position encodings, gated linear units, and grouped query attention. It\\\\u2019s pretrained on a diverse mix of 12T tokens from text and code, allowing for a maximum context length of 32k tokens. This architectural choice enables DBRX to efficiently handle a wide range of tasks with fewer parameters activated per input. Benchmark Performance : DBRX sets new records on various benchmarks, including language understanding, programming, and mathematics, significantly outperforming other leading open models. It is also competitive with or surpasses leading closed models in nearly all considered benchmarks, particularly excelling in programming and mathematical reasoning. The plot below from the paper shows that DBRX outperforms established open source models on language understanding (MMLU), Programming (HumanEval), and Math (GSM8K). Open Access and Integration : DBRX weights for both base and fine-tuned versions are openly available on Hugging Face, facilitating easy access for further experimentation and development. Databricks customers can utilize DBRX via APIs and have the option to pretrain or continue training DBRX-class models using Databricks tools and infrastructure. Training and Inference Efficiency : The paper highlights DBRX\\\\u2019s training and inference efficiency, illustrating that MoE models like DBRX offer significant improvements in compute efficiency. It provides detailed comparisons in training efficiency, showing that models within the DBRX family require fewer FLOPs to reach similar or better performance scores compared to denser models. In terms of inference, DBRX achieves higher throughput than comparable non-MoE models, benefiting from the model\\\\u2019s efficient parameter usage.')]}"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "answer3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create a custom prompt template\n",
    "prompt_template = \"\"\"You are an AI assistant tasked with writing a comprehensive document based on a provided table of contents. \n",
    "Your goal is to generate detailed content for each section and subsection of the document.\n",
    "\n",
    "Table of Contents:\n",
    "{toc}\n",
    "\n",
    "AI Assistant: Write a detailed document covering all the sections and subsections provided in the table of contents above. \n",
    "For each main section (##) and subsection (###), provide comprehensive and informative content. \n",
    "Ensure the content is well-structured, relevant to the topic, and flows logically from one section to the next. \n",
    "Do not include the section or subsection headers in your response, as these will be added separately.\n",
    "\n",
    "If you don't have enough information on a particular topic, state that more research is needed for that specific section or subsection.\n",
    "\n",
    "Begin writing the document now, starting with the content for the first main section.\"\"\"\n",
    "\n",
    "PROMPT = PromptTemplate(\n",
    "    template=prompt_template, input_variables=[\"context\", \"question\"]\n",
    ")\n",
    "\n",
    "# Create a retrieval QA chain\n",
    "qa = RetrievalQA.from_chain_type(\n",
    "    llm=llm,\n",
    "    chain_type=\"stuff\",\n",
    "    retriever=loaded_vectorstore.as_retriever(),\n",
    "    return_source_documents=True,\n",
    "    chain_type_kwargs={\"prompt\": PROMPT}\n",
    ")\n",
    "\n",
    "# query = \"Introduction: What is the purpose of this document?\"\n",
    "# result = qa.invoke({\"query\": query})\n",
    "\n",
    "def notegen(query):\n",
    "    result = qa.invoke({\"query\": query})\n",
    "    return result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "answer = notegen(query=\"#Understanding Large Language Model Architectures, ##1. Fundamentals of LLMs, ###1.1 Definition and Key Concepts, ###1.2 Historical Development of LLMs, ##2. Core Architectural Components, ###2.1 Transformer Architecture, ###2.2 Attention Mechanism, ###2.3 Self-Attention Mechanism, ##3. Training and Fine-Tuning LLMs, ###3.1 Data Preprocessing, ###3.2 Training Process, ###3.3 Fine-Tuning Process, ##4. Applications of LLMs, ###4.1 Natural Language Processing, ###4.2 Text Generation, ###4.3 Machine Translation, ##5. Challenges and Limitations, ###5.1 Ethical Concerns, ###5.2 Bias and Fairness, ###5.3 Computational Resources, ##6. Future Directions, ###6.1 Advancements in LLMs, ###6.2 Research and Development, ###6.3 Potential Applications\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "answer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "prompt_template = \"\"\"You are an AI assistant tasked with writing a comprehensive document based on a provided table of contents. \n",
    "Use the following pieces of context to write detailed sections for the document.\n",
    "If you don't have enough information, state that more research is needed on that topic.\n",
    "\n",
    "Context: {context}\n",
    "\n",
    "Table of Contents:\n",
    "{question}\n",
    "\n",
    "AI Assistant: Write a detailed document covering all the sections and subsections provided in the table of contents above. \n",
    "For each main section (##) and subsection (###), provide comprehensive and informative content. \n",
    "Ensure the content is well-structured, relevant to the topic, and flows logically from one section to the next. \n",
    "Do not include the section or subsection headers in your response, as these will be added separately.\"\"\"\n",
    "\n",
    "PROMPT = PromptTemplate(\n",
    "    template=prompt_template, input_variables=[\"context\", \"question\"]\n",
    ")\n",
    "\n",
    "# Update the qa chain with the new prompt\n",
    "qa = RetrievalQA.from_chain_type(\n",
    "    llm=llm,\n",
    "    chain_type=\"stuff\",\n",
    "    retriever=loaded_vectorstore.as_retriever(),\n",
    "    return_source_documents=True,\n",
    "    chain_type_kwargs={\"prompt\": PROMPT}\n",
    ")\n",
    "\n",
    "def notegen(query):\n",
    "    result = qa.invoke({\"query\": query})\n",
    "    return result\n",
    "\n",
    "# Now you can use notegen as before\n",
    "answer = notegen(query=\"#Understanding Large Language Model Architectures, ##1. Fundamentals of LLMs, ###1.1 Definition and Key Concepts, ###1.2 Historical Development of LLMs, ##2. Core Architectural Components, ###2.1 Transformer Architecture, ###2.2 Attention Mechanism, ###2.3 Self-Attention Mechanism, ##3. Training and Fine-Tuning LLMs, ###3.1 Data Preprocessing, ###3.2 Training Process, ###3.3 Fine-Tuning Process, ##4. Applications of LLMs, ###4.1 Natural Language Processing, ###4.2 Text Generation, ###4.3 Machine Translation, ##5. Challenges and Limitations, ###5.1 Ethical Concerns, ###5.2 Bias and Fairness, ###5.3 Computational Resources, ##6. Future Directions, ###6.1 Advancements in LLMs, ###6.2 Research and Development, ###6.3 Potential Applications\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "answer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "spambots",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}