10.1-phonetics2vec.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import math\n",
    "import glob\n",
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "from tensorflow.contrib.tensorboard.plugins import projector"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "vocab_size: 85 \n",
      "\n"
     ]
    }
   ],
   "source": [
    "import tools.processing as pre\n",
    "\n",
    "batch_size = 256\n",
    "embedding_dimension = 3\n",
    "negative_samples = 32\n",
    "LOG_DIR = \"logs/phone2vec-test\"\n",
    "EPOCHS = 5\n",
    "\n",
    "text = pre.get_text(\"data/phonem-rap-lyrics/phonem_all.txt\")\n",
    "sentences = [text.replace( \"\\n\", \";\" )]\n",
    "\n",
    "vocab = pre.Vocabulary(sentences[0])\n",
    "\n",
    "# Map words to indices\n",
    "index2word = vocab.index2word\n",
    "word2index = vocab.word2index\n",
    "\n",
    "vocabulary_size = len(index2word)\n",
    "print(\"vocab_size: {} \\n\".format(vocabulary_size))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Generate skip-gram pairs for all phonetics\n",
    "### We have chosen a window size of 1 in order to not get topical meanings <br/> So each target word gets combined with its two context words as a tupel"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Generate skip-gram pairs\n",
    "skip_gram_pairs = []\n",
    "for sent in sentences:\n",
    "    tokenized_sent = sent.split()\n",
    "    for i in range(1, len(tokenized_sent) - 1):\n",
    "        word_context_pair = [[word2index[tokenized_sent[i-1]],\n",
    "                              word2index[tokenized_sent[i+1]]],\n",
    "                             word2index[tokenized_sent[i]]]\n",
    "        skip_gram_pairs.append([word_context_pair[1],\n",
    "                                word_context_pair[0][0]])\n",
    "        skip_gram_pairs.append([word_context_pair[1],\n",
    "                                word_context_pair[0][1]])\n",
    "\n",
    "def get_skipgram_batch(batch_size):\n",
    "    instance_indices = list(range(len(skip_gram_pairs)))\n",
    "    np.random.shuffle(instance_indices)\n",
    "    batch = instance_indices[:batch_size]\n",
    "    x = [skip_gram_pairs[i][0] for i in batch]\n",
    "    y = [[skip_gram_pairs[i][1]] for i in batch]\n",
    "    return x, y"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Building the model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### We need to reset the default graph every time we want to retrain our model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# start tensorflow\n",
    "tf.reset_default_graph()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /Users/rene/workspace/Tensorflow/.pyenv/lib/python3.7/site-packages/tensorflow/python/framework/op_def_library.py:263: colocate_with (from tensorflow.python.framework.ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Colocations handled automatically by placer.\n",
      "WARNING:tensorflow:From /Users/rene/workspace/Tensorflow/.pyenv/lib/python3.7/site-packages/tensorflow/python/ops/array_grad.py:425: to_int32 (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use tf.cast instead.\n"
     ]
    }
   ],
   "source": [
    "# Input data, labels\n",
    "train_inputs = tf.placeholder(tf.int32, shape=[batch_size])\n",
    "train_labels = tf.placeholder(tf.int32, shape=[batch_size, 1])\n",
    "\n",
    "# Embedding lookup table currently only implemented in CPU\n",
    "with tf.name_scope(\"embeddings\"):\n",
    "    embeddings = tf.Variable(\n",
    "        tf.random_uniform([vocabulary_size, embedding_dimension],\n",
    "                          -1.0, 1.0), name='embedding')\n",
    "    # This is essentially a lookup table\n",
    "    embed = tf.nn.embedding_lookup(embeddings, train_inputs)\n",
    "\n",
    "# Create variables for the NCE loss\n",
    "nce_weights = tf.Variable(\n",
    "        tf.truncated_normal([vocabulary_size, embedding_dimension],\n",
    "                            stddev=1.0 / math.sqrt(embedding_dimension)))\n",
    "nce_biases = tf.Variable(tf.zeros([vocabulary_size]))\n",
    "\n",
    "\n",
    "loss = tf.reduce_mean(\n",
    "  tf.nn.nce_loss(weights=nce_weights, biases=nce_biases, inputs=embed, labels=train_labels,\n",
    "                 num_sampled=negative_samples, num_classes=vocabulary_size))\n",
    "tf.summary.scalar(\"NCE_loss\", loss)\n",
    "\n",
    "# Learning rate decay\n",
    "global_step = tf.Variable(0, trainable=False)\n",
    "learningRate = tf.train.exponential_decay(learning_rate=0.1,\n",
    "                                          global_step=global_step,\n",
    "                                          decay_steps=1000,\n",
    "                                          decay_rate=0.95,\n",
    "                                          staircase=True)\n",
    "train_step = tf.train.GradientDescentOptimizer(learningRate).minimize(loss)\n",
    "\n",
    "merged = tf.summary.merge_all()\n",
    "\n",
    "saver = tf.train.Saver(keep_checkpoint_every_n_hours=0.5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Starting the training"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### In order to visualize our output we need to create the following: <br/> <ul><li>metadata</li><li>projectorConfiguration</li></ul>\n",
    "#### We are saving our model into two model checkpoints. The first \"embedding\"-checkpoint is just for the embeddings, in case one would forcefully stop the process. <br/> The second \"final-embedding\"-checkpoint is for saving the complete model after all epochs have finished"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Building model from scratch!\n",
      "\n",
      "\n",
      "epoch: 0\n",
      "\n",
      "Loss at 0/1000: 41.19302\n",
      "Loss at 100/1000: 10.40371\n",
      "Loss at 200/1000: 4.51695\n",
      "Loss at 300/1000: 3.83374\n",
      "Loss at 400/1000: 3.72875\n",
      "Loss at 500/1000: 3.82324\n",
      "Loss at 600/1000: 4.29863\n",
      "Loss at 700/1000: 3.78456\n",
      "Loss at 800/1000: 3.39517\n",
      "Loss at 900/1000: 3.93464\n",
      "\n",
      "\n",
      "epoch: 1\n",
      "\n",
      "Loss at 0/1000: 4.12607\n",
      "Loss at 100/1000: 3.55881\n",
      "Loss at 200/1000: 3.67067\n",
      "Loss at 300/1000: 3.41221\n",
      "Loss at 400/1000: 3.25622\n",
      "Loss at 500/1000: 3.28257\n",
      "Loss at 600/1000: 3.39348\n",
      "Loss at 700/1000: 3.27252\n",
      "Loss at 800/1000: 3.21166\n",
      "Loss at 900/1000: 3.63068\n",
      "\n",
      "\n",
      "epoch: 2\n",
      "\n",
      "Loss at 0/1000: 3.74564\n",
      "Loss at 100/1000: 3.19109\n",
      "Loss at 200/1000: 3.24582\n",
      "Loss at 300/1000: 3.28715\n",
      "Loss at 400/1000: 3.30601\n",
      "Loss at 500/1000: 2.87213\n",
      "Loss at 600/1000: 3.46432\n",
      "Loss at 700/1000: 3.20118\n",
      "Loss at 800/1000: 3.81336\n",
      "Loss at 900/1000: 3.28137\n",
      "\n",
      "\n",
      "epoch: 3\n",
      "\n",
      "Loss at 0/1000: 3.30725\n",
      "Loss at 100/1000: 3.55253\n",
      "Loss at 200/1000: 3.44056\n",
      "Loss at 300/1000: 3.39185\n",
      "Loss at 400/1000: 3.30659\n",
      "Loss at 500/1000: 3.42882\n",
      "Loss at 600/1000: 3.07721\n",
      "Loss at 700/1000: 3.00432\n",
      "Loss at 800/1000: 2.89155\n",
      "Loss at 900/1000: 3.24136\n",
      "\n",
      "\n",
      "epoch: 4\n",
      "\n",
      "Loss at 0/1000: 3.52344\n",
      "Loss at 100/1000: 3.21163\n",
      "Loss at 200/1000: 3.31803\n",
      "Loss at 300/1000: 3.34881\n",
      "Loss at 400/1000: 3.09374\n",
      "Loss at 500/1000: 3.77215\n",
      "Loss at 600/1000: 3.37927\n",
      "Loss at 700/1000: 3.36708\n",
      "Loss at 800/1000: 3.48918\n",
      "Loss at 900/1000: 3.21727\n",
      "WARNING:tensorflow:From /Users/rene/workspace/Tensorflow/.pyenv/lib/python3.7/site-packages/tensorflow/python/training/saver.py:966: remove_checkpoint (from tensorflow.python.training.checkpoint_management) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use standard file APIs to delete files with this prefix.\n"
     ]
    }
   ],
   "source": [
    "TRAIN = True\n",
    "with tf.Session() as sess:\n",
    "    sess.run(tf.global_variables_initializer())\n",
    "\n",
    "    train_writer = tf.summary.FileWriter(LOG_DIR, graph=tf.get_default_graph())\n",
    "    \n",
    "    if not os.path.exists(LOG_DIR):\n",
    "        os.makedirs(LOG_DIR, exist_ok=True)\n",
    "            \n",
    "    with open(os.path.join(LOG_DIR, 'metadata.tsv'), \"w\") as metadata:\n",
    "        metadata.write('Name\\tClass\\n')\n",
    "        for k, v in index2word.items():\n",
    "            metadata.write('%s\\t%d\\n' % (v, k))\n",
    "\n",
    "    if glob.glob(LOG_DIR + '/*.meta'):\n",
    "        TRAIN = True # set this value to false, if you don't want to retrain the model\n",
    "        saver = tf.train.import_meta_graph(glob.glob(LOG_DIR + '/*.meta')[0])\n",
    "        saver.restore(sess, os.path.join(LOG_DIR, \"final_embeddings.ckpt\"))\n",
    "        # global_step = sess.run(global_step)\n",
    "        print(\"Restoring an old model and training it further..\")\n",
    "    else:\n",
    "        print(\"Building model from scratch!\")\n",
    "        # global_step = 0\n",
    "\n",
    "    config = projector.ProjectorConfig()\n",
    "    embedding = config.embeddings.add()\n",
    "    embedding.tensor_name = embeddings.name\n",
    "    # Link this tensor to its metadata file (e.g. labels).\n",
    "    embedding.metadata_path = 'metadata.tsv'\n",
    "    projector.visualize_embeddings(train_writer, config)\n",
    "\n",
    "    if TRAIN:\n",
    "        for epoch in range(EPOCHS):\n",
    "            print(f\"\\n\\nepoch: {epoch}\\n\")\n",
    "            \n",
    "            # epoch_steps = (int(len(skip_gram_pairs)/batch_size))\n",
    "            epoch_steps = 1000\n",
    "            for step in range(epoch_steps):\n",
    "                x_batch, y_batch = get_skipgram_batch(batch_size)\n",
    "                summary, _ = sess.run([merged, train_step],\n",
    "                                    feed_dict={train_inputs: x_batch,\n",
    "                                                train_labels: y_batch})\n",
    "                # TODO we would need global_step here in order to get a nice diagram\n",
    "                #      Now every time we start with counting from zero\n",
    "                train_writer.add_summary(summary, step)\n",
    "                \n",
    "                if step % 100 == 0:\n",
    "                    loss_value = sess.run(loss,\n",
    "                                            feed_dict={train_inputs: x_batch,\n",
    "                                                        train_labels: y_batch})\n",
    "                    print(\"Loss at %d/%d: %.5f\" % (step, epoch_steps, loss_value))\n",
    "\n",
    "\n",
    "            saver.save(sess, os.path.join(LOG_DIR, \"embeddings.ckpt\"), epoch)\n",
    "                \n",
    "    saver.save(sess, os.path.join(LOG_DIR, \"final_embeddings.ckpt\"))\n",
    "\n",
    "    # Normalize embeddings before using\n",
    "    norm = tf.sqrt(tf.reduce_sum(tf.square(embeddings), 1, keepdims=True))\n",
    "    normalized_embeddings = embeddings / norm\n",
    "    normalized_embeddings_matrix = sess.run(normalized_embeddings)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Test the similarity of a given word by its cosine distance\n",
    "### The 10 next similar words are displayed"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "AY0 \t 0.99999994\n",
      "UW \t 0.9812705\n",
      "AO2 \t 0.9131582\n",
      "EH \t 0.7681473\n",
      "EY \t 0.7136153\n",
      "N \t 0.713245\n",
      "NG \t 0.6829777\n",
      "G \t 0.672083\n",
      "ER2 \t 0.6545277\n",
      "K \t 0.6018664\n"
     ]
    }
   ],
   "source": [
    "ref_word = normalized_embeddings_matrix[word2index[\"AY0\"]]\n",
    "\n",
    "cosine_dists = np.dot(normalized_embeddings_matrix, ref_word)\n",
    "ff = np.argsort(cosine_dists)[::-1][0:10]\n",
    "for f in ff:\n",
    "    print(index2word[f], \"\\t\", cosine_dists[f])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## At last the encoded embedding matrix is going to be saved as a text-file "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "# saving embedding matrix to file\n",
    "with open(os.path.join(LOG_DIR, \"embedding.txt\"), 'w') as f:\n",
    "    for i in range(vocabulary_size):\n",
    "      embed = normalized_embeddings_matrix[i, :]\n",
    "      word = index2word[i]\n",
    "      f.write('%s %s\\n' % (word, ' '.join(map(str, embed))))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}