LogisticRegressionScratch.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 126,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "import mxnet\n",
    "from mxnet import autograd, nd, gluon\n",
    "from mxnet.gluon.data.vision import transforms"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 127,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_dataloader(batch_size, num_workers):\n",
    "    transformer = transforms.Compose([\n",
    "        transforms.ToTensor()\n",
    "    ])\n",
    "    train = gluon.data.vision.datasets.FashionMNIST(train=True)\n",
    "    train = train.transform_first(transformer)\n",
    "    train_iter = gluon.data.DataLoader(train, batch_size, shuffle=True, num_workers=num_workers)\n",
    "    test = gluon.data.vision.datasets.FashionMNIST(train=False)\n",
    "    test = test.transform_first(transformer)\n",
    "    test_iter = gluon.data.DataLoader(test, batch_size, shuffle=True, num_workers=num_workers)\n",
    "    return train_iter, test_iter"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 128,
   "metadata": {},
   "outputs": [],
   "source": [
    "def softmax(X):\n",
    "    X_exp = X.exp()\n",
    "    normalization_constant = X_exp.sum(axis=1, keepdims=True)\n",
    "    return X_exp / normalization_constant"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 168,
   "metadata": {},
   "outputs": [],
   "source": [
    "def cross_entropy(y_hat, y):\n",
    "    return -nd.pick(y_hat, y).log()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 169,
   "metadata": {},
   "outputs": [],
   "source": [
    "def sgd(params, lr, batch_size):\n",
    "    for param in params:\n",
    "        param[:] = param - (lr / batch_size) * param.grad"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 170,
   "metadata": {},
   "outputs": [],
   "source": [
    "def accuracy(y_hat, y):\n",
    "    return (y_hat.argmax(axis=1)==y.astype('float32')).sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 171,
   "metadata": {},
   "outputs": [],
   "source": [
    "def evaluate_acc(net,W, b, num_features, data_iter):\n",
    "    accumulator = 0\n",
    "    size = 0\n",
    "    for X, y in data_iter:\n",
    "        y_hat = net(X, W, b, num_features)\n",
    "        accumulator += accuracy(y_hat, y)\n",
    "        size += len(y)\n",
    "    return accumulator / size"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 172,
   "metadata": {},
   "outputs": [],
   "source": [
    "def net(X, W, b, num_features):\n",
    "    return softmax(nd.dot(X.reshape(-1, num_features), W) + b)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 173,
   "metadata": {},
   "outputs": [],
   "source": [
    "batch_size = 256\n",
    "num_workers = 4\n",
    "train_iter, test_iter = get_dataloader(batch_size, num_workers)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 174,
   "metadata": {},
   "outputs": [],
   "source": [
    "num_inputs = 28 * 28\n",
    "num_outputs = 10\n",
    "W = nd.random.normal(scale=0.01, shape=(num_inputs, num_outputs))\n",
    "b = nd.zeros(num_outputs)\n",
    "W.attach_grad()\n",
    "b.attach_grad()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 175,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 0, acc: 0.805417\n",
      "Epoch 1, acc: 0.820217\n",
      "Epoch 2, acc: 0.829133\n",
      "Epoch 3, acc: 0.834200\n",
      "Epoch 4, acc: 0.839000\n",
      "Epoch 5, acc: 0.841550\n",
      "Epoch 6, acc: 0.844400\n",
      "Epoch 7, acc: 0.845233\n",
      "Epoch 8, acc: 0.846100\n",
      "Epoch 9, acc: 0.846767\n"
     ]
    }
   ],
   "source": [
    "num_epochs = 10\n",
    "lr = 0.1\n",
    "for epochs in range(num_epochs):\n",
    "    for X, y in train_iter:\n",
    "        with autograd.record():\n",
    "            y_hat = net(X, W, b, num_inputs)\n",
    "            l = cross_entropy(y_hat, y)\n",
    "        l.backward()\n",
    "#         sgd([W, b], lr, batch_size)\n",
    "        W[:] = W - (lr / batch_size) * W.grad\n",
    "        b[:] = b - (lr / batch_size) * b.grad\n",
    "    epoch_acc = evaluate_acc(net, W, b, num_inputs, train_iter)\n",
    "    print(\"Epoch %d, acc: %f\" % (epochs, epoch_acc.asscalar()))\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 154,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(256, 1, 28, 28)\n",
      "(256,)\n",
      "(256,)\n",
      "\n",
      "[0.11328125]\n",
      "<NDArray 1 @cpu(0)>\n"
     ]
    }
   ],
   "source": [
    "for X, y in train_iter:\n",
    "    print(X.shape)\n",
    "    print(y.shape)\n",
    "    y_hat = net(X, W, b, num_inputs)\n",
    "    print(cross_entropy(y_hat, y).shape)\n",
    "    print(accuracy(y_hat, y)/len(y))\n",
    "    break"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}