encode.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Usage of `spm_encode`\n",
    "\n",
    "This is an example of using spm_encode."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/home/fsuser/haoyiyang/sentencepiece\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/home/fsuser/haoyiyang/sentencepiece/build\n",
      "-- VERSION: 0.2.00\n",
      "-- Found the following ICU libraries:\n",
      "--   i18n (required)\n",
      "--   data (required)\n",
      "--   uc (required)\n",
      "-- Found TCMalloc: /usr/lib/x86_64-linux-gnu/libtcmalloc_minimal.so\n",
      "-- Configuring done\n",
      "-- Generating done\n",
      "-- Build files have been written to: /home/fsuser/haoyiyang/sentencepiece/build\n",
      "\u001b[35m\u001b[1mConsolidate compiler generated dependencies of target sentencepiece_train-static\u001b[0m\n",
      "\u001b[35m\u001b[1mConsolidate compiler generated dependencies of target sentencepiece\u001b[0m\n",
      "\u001b[35m\u001b[1mConsolidate compiler generated dependencies of target sentencepiece-static\u001b[0m\n",
      "[  9%] Built target sentencepiece_train-static\n",
      "[ 70%] Built target sentencepiece-static\n",
      "[ 79%] Built target sentencepiece\n",
      "\u001b[35m\u001b[1mConsolidate compiler generated dependencies of target spm_encode\u001b[0m\n",
      "\u001b[35m\u001b[1mConsolidate compiler generated dependencies of target spm_decode\u001b[0m\n",
      "\u001b[35m\u001b[1mConsolidate compiler generated dependencies of target spm_bpe_cache_merge\u001b[0m\n",
      "\u001b[35m\u001b[1mConsolidate compiler generated dependencies of target spm_export_vocab\u001b[0m\n",
      "\u001b[35m\u001b[1mConsolidate compiler generated dependencies of target sentencepiece_train\u001b[0m\n",
      "[ 80%] \u001b[32mBuilding CXX object src/CMakeFiles/spm_encode.dir/spm_encode_main.cc.o\u001b[0m\n",
      "[ 82%] Built target spm_decode\n",
      "[ 84%] Built target spm_bpe_cache_merge\n",
      "[ 86%] Built target spm_export_vocab\n",
      "[ 95%] Built target sentencepiece_train\n",
      "\u001b[35m\u001b[1mConsolidate compiler generated dependencies of target spm_normalize\u001b[0m\n",
      "\u001b[35m\u001b[1mConsolidate compiler generated dependencies of target spm_train\u001b[0m\n",
      "\u001b[35m\u001b[1mConsolidate compiler generated dependencies of target compile_charsmap\u001b[0m\n",
      "[ 97%] Built target spm_normalize\n",
      "[ 99%] Built target spm_train\n",
      "[100%] Built target compile_charsmap\n",
      "[100%] \u001b[32m\u001b[1mLinking CXX executable spm_encode\u001b[0m\n",
      "[100%] Built target spm_encode\n",
      "/home/fsuser/haoyiyang/sentencepiece\n"
     ]
    }
   ],
   "source": [
    "# Build the project\n",
    "!mkdir -p build\n",
    "%cd build\n",
    "!cmake -D SPM_USE_EXTERNAL_ABSL=off -D SPM_ENABLE_TCMALLOC=on -D SPM_ENABLE_NFKC_COMPILE=on ..\n",
    "# Alternatively, debug build:\n",
    "# !cmake \\\n",
    "#       -DCMAKE_BUILD_TYPE=Debug -DCMAKE_C_FLAGS_DEBUG=\"-g -O0\" \\\n",
    "#       -DCMAKE_CXX_FLAGS_DEBUG=\"-g -O0\" \\\n",
    "#       -D SPM_USE_EXTERNAL_ABSL=off -D SPM_ENABLE_TCMALLOC=on -D SPM_ENABLE_NFKC_COMPILE=on ..\n",
    "!make -j\n",
    "%cd .."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Document with code$\n",
      "line 1$\n",
      "^Aint x;$\n",
      "float y;^B$\n",
      "line 2$\n",
      "^@^CFile path: code.cpp^D^Aint x;$\n",
      "float y;^B^@"
     ]
    }
   ],
   "source": [
    "!printf 'Document with code\\nline 1\\n\\1int x;\\nfloat y;\\2\\nline 2\\n\\0\\3File path: code.cpp\\4\\1int x;\\nfloat y;\\2\\0' > example.in\n",
    "!cat -A example.in"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "spm_encode_main.cc(314) LOG(INFO) Encoded 2 sentences\n",
      "spm_encode_main.cc(341) LOG(INFO) Encoded 2 sentences\n"
     ]
    }
   ],
   "source": [
    "!./build/src/spm_encode --model=/home/fsuser/datasets/100k/pile_100k.sp.model --output_format=bid --input example.in --output example.bid --new_line_delim=0\\\n",
    "    --verbatim_control_char=1 --code_block_end=2 --code_meta_block_begin=3 --code_meta_block_end=4"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "n_sentences = 2\n",
      "sentence_sizes = [22, 17]\n",
      "sentence_ids[0] = [13113, 356, 1868, 99340, 1183, 99320, 99347, 99340, 4, 555, 1166, 462, 8540, 357, 99391, 5, 678, 1183, 99320, 99351, 99340, 3]\n",
      "sentence_ids[1] = [6, 7596, 1698, 99374, 1868, 99339, 12172, 7, 4, 555, 1166, 462, 8540, 357, 99391, 5, 3]\n"
     ]
    }
   ],
   "source": [
    "with open(\"example.bid\", \"rb\") as f:\n",
    "    contents = f.read()\n",
    "n_sentences = int.from_bytes(contents[-8:], \"little\")\n",
    "print(\"n_sentences =\", n_sentences)\n",
    "sentence_sizes = []\n",
    "for i in range(n_sentences):\n",
    "    buffer = contents[-8 - (i+1) * 4: -8 - i*4]\n",
    "    sentence_size = int.from_bytes(buffer, \"little\")\n",
    "    sentence_sizes.append(sentence_size)\n",
    "sentence_sizes = list(reversed(sentence_sizes))\n",
    "print(\"sentence_sizes =\", sentence_sizes)\n",
    "ptr = 0\n",
    "sentence_ids = []\n",
    "for sentence_i, sentence_size in enumerate(sentence_sizes):\n",
    "    buffer = contents[ptr: ptr + sentence_size * 4]\n",
    "    ids = []\n",
    "    for i in range(0, len(buffer), 4):\n",
    "        ids.append(int.from_bytes(buffer[i: i + 4], \"little\"))\n",
    "    sentence_ids.append(ids)\n",
    "    print(f\"sentence_ids[{sentence_i}] =\", ids)\n",
    "    ptr += sentence_size * 4"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Load vocab size of 100000\n"
     ]
    }
   ],
   "source": [
    "vocab_file = \"/home/fsuser/datasets/100k/pile_100k.sp.vocab\"\n",
    "tokens = []\n",
    "with open(vocab_file, \"r\", encoding=\"utf-8\") as f:\n",
    "    buffer = \"\"\n",
    "    for line in f:\n",
    "        buffer += line\n",
    "        if \"\\t\" in line:\n",
    "            tokens.append(buffer[:buffer.index(\"\\t\")])\n",
    "            buffer = \"\"\n",
    "print(\"Load vocab size of\", len(tokens))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "====================\n",
      "▁Document ▁with ▁code \n",
      " line ▁ 1 \n",
      " <0x01> ▁int ▁x ;\n",
      " float ▁y ; <0x02> ▁\n",
      " line ▁ 2 \n",
      " <0x00>\n",
      "====================\n",
      "<0x03> ▁File ▁path : ▁code . cpp <0x04> <0x01> ▁int ▁x ;\n",
      " float ▁y ; <0x02> <0x00>\n"
     ]
    }
   ],
   "source": [
    "for ids in sentence_ids:\n",
    "    print(\"=\" * 20)\n",
    "    print(\" \".join(tokens[i] for i in ids))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}