examples/IRR/DINA.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "source": [
    "# Item Response Ranking for DINA\n",
    "\n",
    "This notebook will show you how to train and use the IRR-DINA.\n",
    "Refer to [IRR doc](../../docs/IRR.md) for more details.\n",
    "First, we will show how to get the data (here we use a0910 as the dataset).\n",
    "Then we will show how to train a IRR-DINA and perform the parameters persistence.\n",
    "At last, we will show how to load the parameters from the file and evaluate on the test dataset."
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%% md\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "reading records from ../../data/a0910/item.csv: 100%|██████████| 19529/19529 [00:00<00:00, 55368.84it/s]\n",
      "rating2triplet: 100%|██████████| 17051/17051 [00:15<00:00, 1107.24it/s]\n"
     ]
    },
    {
     "data": {
      "text/plain": "(<longling.lib.iterator.AsyncLoopIter at 0x1fce3084dc0>,\n <torch.utils.data.dataloader.DataLoader at 0x1fcd3e2df10>,\n <torch.utils.data.dataloader.DataLoader at 0x1fce305e550>)"
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import logging\n",
    "from longling.lib.structure import AttrDict\n",
    "from longling import set_logging_info\n",
    "from EduCDM.IRR import pair_etl as etl, point_etl as vt_etl, extract_item\n",
    "\n",
    "set_logging_info()\n",
    "\n",
    "params = AttrDict(\n",
    "    batch_size=256,\n",
    "    n_neg=10,\n",
    "    n_imp=10,\n",
    "    logger=logging.getLogger(),\n",
    "    hyper_params={\"user_num\": 4164, \"knowledge_num\": 123}\n",
    ")\n",
    "item_knowledge = extract_item(\"../../data/a0910/item.csv\", params[\"hyper_params\"][\"knowledge_num\"], params)\n",
    "train_data, train_df = etl(\"../../data/a0910/train.csv\", item_knowledge, params)\n",
    "valid_data, _ = vt_etl(\"../../data/a0910/valid.csv\", item_knowledge, params)\n",
    "test_data, _ = vt_etl(\"../../data/a0910/test.csv\", item_knowledge, params)\n",
    "\n",
    "train_data, valid_data, test_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "outputs": [
    {
     "data": {
      "text/plain": "        user_id  item_id  score\n0          1615    12977    1.0\n1           782    13124    0.0\n2          1084    16475    0.0\n3           593     8690    0.0\n4           127    14225    1.0\n...         ...      ...    ...\n186044     2280     6019    0.0\n186045      121        2    1.0\n186046      601     5425    1.0\n186047      573     2412    0.0\n186048       60     2969    1.0\n\n[186049 rows x 3 columns]",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>user_id</th>\n      <th>item_id</th>\n      <th>score</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>1615</td>\n      <td>12977</td>\n      <td>1.0</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>782</td>\n      <td>13124</td>\n      <td>0.0</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>1084</td>\n      <td>16475</td>\n      <td>0.0</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>593</td>\n      <td>8690</td>\n      <td>0.0</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>127</td>\n      <td>14225</td>\n      <td>1.0</td>\n    </tr>\n    <tr>\n      <th>...</th>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n    </tr>\n    <tr>\n      <th>186044</th>\n      <td>2280</td>\n      <td>6019</td>\n      <td>0.0</td>\n    </tr>\n    <tr>\n      <th>186045</th>\n      <td>121</td>\n      <td>2</td>\n      <td>1.0</td>\n    </tr>\n    <tr>\n      <th>186046</th>\n      <td>601</td>\n      <td>5425</td>\n      <td>1.0</td>\n    </tr>\n    <tr>\n      <th>186047</th>\n      <td>573</td>\n      <td>2412</td>\n      <td>0.0</td>\n    </tr>\n    <tr>\n      <th>186048</th>\n      <td>60</td>\n      <td>2969</td>\n      <td>1.0</td>\n    </tr>\n  </tbody>\n</table>\n<p>186049 rows × 3 columns</p>\n</div>"
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_df"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 0: 727it [01:00, 12.00it/s]\n",
      "evaluating: 100%|██████████| 101/101 [00:00<00:00, 151.21it/s]\n",
      "formatting item df: 100%|██████████| 10415/10415 [00:00<00:00, 13243.30it/s]\n",
      "ranking metrics: 10415it [00:14, 718.34it/s]\n",
      "Epoch 1: 100%|██████████| 727/727 [01:05<00:00, 11.05it/s]\n",
      "evaluating: 100%|██████████| 101/101 [00:00<00:00, 130.40it/s]\n",
      "formatting item df: 100%|██████████| 10415/10415 [00:00<00:00, 11689.41it/s]\n",
      "ranking metrics: 10415it [00:15, 683.86it/s]\n",
      "INFO:root:save parameters to IRR-DINA.params\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Epoch 0] Loss: 2.625543, PointLoss: 0.766550, PairLoss: 4.484537\n",
      "[Epoch 0]\n",
      "      ndcg@k  precision@k  recall@k      f1@k     len@k  support@k\n",
      "1   1.000000     0.695919  0.486584  0.540378  1.000000      10415\n",
      "3   0.894090     0.678829  0.741237  0.689378  1.906961      10415\n",
      "5   0.895159     0.675855  0.796159  0.713132  2.229573      10415\n",
      "10  0.894894     0.674277  0.816401  0.720339  2.423428      10415\n",
      "auc: 0.856217\tmap: 0.884234\tmrr: 0.918452\tcoverage_error: 3.194929\tranking_loss: 0.406348\tlen: 2.458569\tsupport: 10415\n",
      "[Epoch 1] Loss: 2.555724, PointLoss: 0.735666, PairLoss: 4.375782\n",
      "[Epoch 1]\n",
      "      ndcg@k  precision@k  recall@k      f1@k     len@k  support@k\n",
      "1   1.000000     0.697552  0.487173  0.541213  1.000000      10415\n",
      "3   0.895660     0.680493  0.742846  0.690920  1.906961      10415\n",
      "5   0.896209     0.676374  0.796789  0.713664  2.229573      10415\n",
      "10  0.895947     0.674335  0.816513  0.720412  2.423428      10415\n",
      "auc: 0.859627\tmap: 0.884662\tmrr: 0.919965\tcoverage_error: 3.191404\tranking_loss: 0.402945\tlen: 2.458569\tsupport: 10415\n"
     ]
    }
   ],
   "source": [
    "from EduCDM.IRR import DINA\n",
    "\n",
    "cdm = DINA(\n",
    "    4163 + 1,\n",
    "    17746 + 1,\n",
    "    123,\n",
    "    ste=True\n",
    ")\n",
    "cdm.train(\n",
    "    train_data,\n",
    "    valid_data,\n",
    "    epoch=2,\n",
    ")\n",
    "cdm.save(\"IRR-DINA.params\")"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:root:load parameters from IRR-DINA.params\n",
      "evaluating: 100%|██████████| 218/218 [00:01<00:00, 146.09it/s]\n",
      "formatting item df: 100%|██████████| 13682/13682 [00:01<00:00, 10267.90it/s]\n",
      "ranking metrics: 13682it [00:25, 538.15it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "      ndcg@k  precision@k  recall@k      f1@k     len@k  support@k\n",
      "1   1.000000     0.694051  0.381306  0.448280  1.000000      13682\n",
      "3   0.868723     0.680298  0.672331  0.643307  2.268528      13682\n",
      "5   0.872068     0.673917  0.775802  0.695325  2.981582      13682\n",
      "10  0.871886     0.669289  0.846420  0.725201  3.723652      13682\n",
      "auc: 0.794464\tmap: 0.835652\tmrr: 0.888232\tcoverage_error: 4.901166\tranking_loss: 0.456207\tlen: 4.075428\tsupport: 13682\n"
     ]
    }
   ],
   "source": [
    "cdm.load(\"IRR-DINA.params\")\n",
    "print(cdm.eval(test_data))"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}