Model_training.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2.1.0\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import numpy as np \n",
    "import tensorflow as tf\n",
    "from transformers import TFRobertaModel\n",
    "from tensorflow.keras.layers import Input\n",
    "from tensorflow.keras.models import Model\n",
    "from tensorflow.keras.layers import concatenate,Dense,Dot,BatchNormalization,Dropout\n",
    "from tokenizers import ByteLevelBPETokenizer,processors,Tokenizer\n",
    "import random\n",
    "random.seed(0)\n",
    "np.random.seed(0)\n",
    "from transformers import RobertaConfig\n",
    "from tqdm import tqdm\n",
    "import time\n",
    "print(tf.__version__)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 56,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tf.test.is_gpu_available()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Index(['log1', 'log2', 'common'], dtype='object')\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.5/dist-packages/ipykernel_launcher.py:4: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  after removing the cwd from sys.path.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       " 1    37800\n",
       "-1    30502\n",
       "Name: common, dtype: int64"
      ]
     },
     "execution_count": 106,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df=pd.read_csv(\"pairs_data.csv\")\n",
    "print(df.columns)\n",
    "df['common'][df['common']==0]=-1\n",
    "df.common.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(68302, 3)"
      ]
     },
     "execution_count": 58,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['./tokenizer-vocab.json', './tokenizer-merges.txt']"
      ]
     },
     "execution_count": 59,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tokenizer = ByteLevelBPETokenizer()\n",
    "tokenizer.train(files=['training_text.txt'], vocab_size=25000, min_frequency=15, special_tokens=[\n",
    "    \"<s>\",\n",
    "    \"<pad>\",\n",
    "    \"</s>\",\n",
    "    \"<unk>\",\n",
    "    \"<mask>\",\n",
    "])\n",
    "tokenizer.post_processor = processors.RobertaProcessing(\n",
    "  sep=(\"</s>\", tokenizer.token_to_id(\"</s>\")),\n",
    "  cls=(\"<s>\", tokenizer.token_to_id(\"<s>\")),\n",
    ")\n",
    "tokenizer.save(\".\", \"tokenizer\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [],
   "source": [
    "MAX_LEN=200"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [],
   "source": [
    "tokenizer.enable_truncation(max_length=MAX_LEN)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [],
   "source": [
    "def generateIds(output):\n",
    "    ids=output.ids\n",
    "    ids=ids[0:MAX_LEN-2]\n",
    "    ids=[tokenizer.token_to_id('<s>')]+ids+[tokenizer.token_to_id('</s>')]\n",
    "    attentionmask=[1]*len(ids)\n",
    "    \n",
    "    final_ids=ids+[tokenizer.token_to_id('<pad>')]*(MAX_LEN-len(ids))\n",
    "    final_attention_mask=attentionmask+[0]*(MAX_LEN-len(ids))\n",
    "    return final_ids,final_attention_mask\n",
    "\n",
    "\n",
    "def tokenize(x):\n",
    "    output=tokenizer.encode(x)\n",
    "    final_ids,final_attention_mask=generateIds(output)\n",
    "    return tf.convert_to_tensor(final_ids,tf.int64),tf.convert_to_tensor(final_attention_mask,tf.int64)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [],
   "source": [
    "def batchTokenize(data):\n",
    "    toks=[]\n",
    "    atts=[]\n",
    "    for i in tqdm(data):\n",
    "        x,y=tokenize(i)\n",
    "        toks.append(tf.reshape(x,[1,MAX_LEN]))\n",
    "        atts.append(tf.reshape(y,[1,MAX_LEN]))\n",
    "    toks=tf.concat(toks,0)\n",
    "    atts=tf.concat(atts,0)\n",
    "    print(toks.shape)\n",
    "    print(atts.shape)\n",
    "    return (toks,atts)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 68302/68302 [00:37<00:00, 1803.88it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(68302, 200)\n",
      "(68302, 200)\n",
      "CPU times: user 45.4 s, sys: 4.44 s, total: 49.8 s\n",
      "Wall time: 38.2 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "output=batchTokenize(df.log1)\n",
    "toks1_input=output[0]\n",
    "atts1_input=output[1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 68302/68302 [00:36<00:00, 1855.16it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(68302, 200)\n",
      "(68302, 200)\n",
      "CPU times: user 44.5 s, sys: 4.16 s, total: 48.7 s\n",
      "Wall time: 37.1 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "output=batchTokenize(df.log2)\n",
    "toks2_input=output[0]\n",
    "atts2_input=output[1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(68302, 200) (68302, 200) (68302, 200) (68302, 200)\n"
     ]
    }
   ],
   "source": [
    "print(toks1_input.shape,atts1_input.shape,toks2_input.shape,atts2_input.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "metadata": {},
   "outputs": [],
   "source": [
    "config= RobertaConfig()\n",
    "config.num_hidden_layers=2\n",
    "config.attention_probs_dropout_prob=0.3\n",
    "config.vocab_size=tokenizer.get_vocab_size()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "metadata": {},
   "outputs": [],
   "source": [
    "roberta = TFRobertaModel(config)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "RobertaConfig {\n",
       "  \"architectures\": null,\n",
       "  \"attention_probs_dropout_prob\": 0.3,\n",
       "  \"bos_token_id\": null,\n",
       "  \"do_sample\": false,\n",
       "  \"eos_token_ids\": null,\n",
       "  \"finetuning_task\": null,\n",
       "  \"hidden_act\": \"gelu\",\n",
       "  \"hidden_dropout_prob\": 0.1,\n",
       "  \"hidden_size\": 768,\n",
       "  \"id2label\": {\n",
       "    \"0\": \"LABEL_0\",\n",
       "    \"1\": \"LABEL_1\"\n",
       "  },\n",
       "  \"initializer_range\": 0.02,\n",
       "  \"intermediate_size\": 3072,\n",
       "  \"is_decoder\": false,\n",
       "  \"label2id\": {\n",
       "    \"LABEL_0\": 0,\n",
       "    \"LABEL_1\": 1\n",
       "  },\n",
       "  \"layer_norm_eps\": 1e-12,\n",
       "  \"length_penalty\": 1.0,\n",
       "  \"max_length\": 20,\n",
       "  \"max_position_embeddings\": 512,\n",
       "  \"model_type\": \"roberta\",\n",
       "  \"num_attention_heads\": 12,\n",
       "  \"num_beams\": 1,\n",
       "  \"num_hidden_layers\": 2,\n",
       "  \"num_labels\": 2,\n",
       "  \"num_return_sequences\": 1,\n",
       "  \"output_attentions\": false,\n",
       "  \"output_hidden_states\": false,\n",
       "  \"output_past\": true,\n",
       "  \"pad_token_id\": null,\n",
       "  \"pruned_heads\": {},\n",
       "  \"repetition_penalty\": 1.0,\n",
       "  \"temperature\": 1.0,\n",
       "  \"top_k\": 50,\n",
       "  \"top_p\": 1.0,\n",
       "  \"torchscript\": false,\n",
       "  \"type_vocab_size\": 2,\n",
       "  \"use_bfloat16\": false,\n",
       "  \"vocab_size\": 15345\n",
       "}"
      ]
     },
     "execution_count": 112,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "config"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 113,
   "metadata": {},
   "outputs": [],
   "source": [
    "target_values=df.common"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 114,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(None, 1)\n"
     ]
    }
   ],
   "source": [
    "toks1=Input(shape=(MAX_LEN,), dtype='int64')\n",
    "atts1=Input(shape=(MAX_LEN,),dtype=\"int64\")\n",
    "out1=roberta(inputs={'input_ids':toks1,'attention_mask':atts1})\n",
    "toks2=Input(shape=(MAX_LEN,), dtype='int64')\n",
    "atts2=Input(shape=(MAX_LEN,),dtype=\"int64\")\n",
    "out2=roberta(inputs={'input_ids':toks2,'attention_mask':atts2})\n",
    "mean1=tf.reduce_mean(out1[0],1)\n",
    "mean2=tf.reduce_mean(out2[0],1)\n",
    "\n",
    "#########Comment this block if objective is cosine similarity calculation\n",
    "cosine_similarity=Dot(axes=1,normalize=True)\n",
    "preds=cosine_similarity([mean1,mean2])\n",
    "\n",
    "#######Uncomment This block if objective is classification\n",
    "\n",
    "# diff=tf.math.subtract(mean1,mean2)\n",
    "# diff=tf.abs(diff)\n",
    "# merged = concatenate([mean1,mean2,diff])\n",
    "# merged = BatchNormalization()(merged)\n",
    "# merged = Dropout(0.1)(merged)\n",
    "# preds = Dense(1, activation='sigmoid')(merged)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 115,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = Model(inputs=[toks1,atts1,toks2,atts2], outputs=preds)\n",
    "# model.compile(loss='binary_crossentropy', optimizer='nadam', metrics=['acc'])\n",
    "model.compile(loss='mse', optimizer='nadam', metrics=['mse'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 116,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 61471 samples, validate on 6831 samples\n",
      "Epoch 1/2\n",
      "WARNING:tensorflow:Gradients do not exist for variables ['tf_roberta_model_3/roberta/pooler/dense/kernel:0', 'tf_roberta_model_3/roberta/pooler/dense/bias:0'] when minimizing the loss.\n",
      "WARNING:tensorflow:Gradients do not exist for variables ['tf_roberta_model_3/roberta/pooler/dense/kernel:0', 'tf_roberta_model_3/roberta/pooler/dense/bias:0'] when minimizing the loss.\n",
      "61471/61471 [==============================] - 318s 5ms/sample - loss: 0.4187 - mse: 0.4187 - val_loss: 0.0598 - val_mse: 0.0598\n",
      "Epoch 2/2\n",
      "61471/61471 [==============================] - 313s 5ms/sample - loss: 0.0629 - mse: 0.0629 - val_loss: 7.9484e-05 - val_mse: 7.9484e-05\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<tensorflow.python.keras.callbacks.History at 0x7fce0b7f6080>"
      ]
     },
     "execution_count": 116,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.fit([toks1_input,atts1_input,toks2_input,atts2_input],target_values,\n",
    "                  epochs=2, batch_size=64,shuffle=True,validation_split=0.1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 126,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Assets written to: /home/opsmxuser/SBERT/model/assets\n"
     ]
    }
   ],
   "source": [
    "model.save(\"SBERT/model/\",include_optimizer=False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}