Month 5 Part I - Capstone Assignment II - Metrics Calculation.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Import libraries and data\n",
    "\n",
    "Dataset was obtained in the capstone project description (direct link [here](https://d3c33hcgiwev3.cloudfront.net/_429455574e396743d399f3093a3cc23b_capstone.zip?Expires=1530403200&Signature=FECzbTVo6TH7aRh7dXXmrASucl~Cy5mlO94P7o0UXygd13S~Afi38FqCD7g9BOLsNExNB0go0aGkYPtodekxCGblpc3I~R8TCtWRrys~2gciwuJLGiRp4CfNtfp08sFvY9NENaRb6WE2H4jFsAo2Z2IbXV~llOJelI3k-9Waj~M_&Key-Pair-Id=APKAJLTNE6QMUY6HBC5A)) and splited manually in separated csv files. They were stored at my personal github account (folder link [here](https://github.com/caiomiyashiro/RecommenderSystemsNotebooks/tree/master/data/capstone)) and you can download and paste inside your working directory in order for this notebook to run."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Preprocess data\n",
    "\n",
    "Float data came with ',' in the csv and python works with '.', so it treated the number as text. In order to convert them to numbers, I first replaced all the commas by punct and then converted the columns to float."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "items.shape = (200, 7)\n",
      "actual_ratings.shape = (200, 100)\n",
      "content_based.shape = (200, 100)\n",
      "user_user.shape = (200, 100)\n",
      "item_item.shape = (200, 100)\n",
      "matrix_fact.shape = (200, 100)\n",
      "pers_bias.shape = (200, 100)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>64</th>\n",
       "      <th>65</th>\n",
       "      <th>75</th>\n",
       "      <th>79</th>\n",
       "      <th>83</th>\n",
       "      <th>112</th>\n",
       "      <th>252</th>\n",
       "      <th>271</th>\n",
       "      <th>301</th>\n",
       "      <th>305</th>\n",
       "      <th>...</th>\n",
       "      <th>3411</th>\n",
       "      <th>3430</th>\n",
       "      <th>3524</th>\n",
       "      <th>3533</th>\n",
       "      <th>3625</th>\n",
       "      <th>3902</th>\n",
       "      <th>3991</th>\n",
       "      <th>4047</th>\n",
       "      <th>4342</th>\n",
       "      <th>4462</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>item</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>4.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>35</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>4.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>41</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>45</th>\n",
       "      <td>NaN</td>\n",
       "      <td>5.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>5.0</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 100 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      64   65   75   79  83  112  252  271  301  305  ...   3411  3430  3524  \\\n",
       "item                                                  ...                      \n",
       "24   NaN  NaN  NaN  NaN NaN  NaN  4.0  5.0  4.0  5.0  ...    NaN   NaN   NaN   \n",
       "30   NaN  NaN  NaN  NaN NaN  NaN  NaN  NaN  NaN  NaN  ...    NaN   NaN   NaN   \n",
       "35   NaN  NaN  NaN  4.0 NaN  NaN  NaN  NaN  NaN  NaN  ...    NaN   NaN   NaN   \n",
       "41   NaN  NaN  NaN  NaN NaN  NaN  NaN  NaN  NaN  NaN  ...    NaN   NaN   NaN   \n",
       "45   NaN  5.0  4.0  5.0 NaN  NaN  NaN  NaN  NaN  5.0  ...    NaN   NaN   NaN   \n",
       "\n",
       "      3533  3625  3902  3991  4047  4342  4462  \n",
       "item                                            \n",
       "24     NaN   NaN   NaN   NaN   NaN   NaN   NaN  \n",
       "30     NaN   NaN   NaN   NaN   NaN   NaN   NaN  \n",
       "35     NaN   NaN   NaN   NaN   NaN   NaN   NaN  \n",
       "41     NaN   NaN   NaN   NaN   NaN   NaN   NaN  \n",
       "45     NaN   NaN   NaN   NaN   NaN   NaN   NaN  \n",
       "\n",
       "[5 rows x 100 columns]"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "items = pd.read_csv('data/capstone/Capstone Data - Office Products - Items.csv', index_col=0) \n",
    "actual_ratings = pd.read_csv('data/capstone/Capstone Data - Office Products - Ratings.csv', index_col=0) \n",
    "\n",
    "content_based = pd.read_csv('data/capstone/Capstone Data - Office Products - CBF.csv', index_col=0)\n",
    "user_user = pd.read_csv('data/capstone/Capstone Data - Office Products - User-User.csv', index_col=0)\n",
    "item_item = pd.read_csv('data/capstone/Capstone Data - Office Products - Item-Item.csv', index_col=0)\n",
    "matrix_fact = pd.read_csv('data/capstone/Capstone Data - Office Products - MF.csv', index_col=0)\n",
    "pers_bias = pd.read_csv('data/capstone/Capstone Data - Office Products - PersBias.csv', index_col=0)\n",
    "\n",
    "items[['Availability','Price']] = items[['Availability','Price']].apply(lambda col: col.apply(lambda elem: str(elem).replace(',', '.'))).astype(float)\n",
    "\n",
    "# preprocess\n",
    "content_based = content_based.apply(lambda col: col.apply(lambda elem: str(elem).replace(',', '.'))).astype(float)\n",
    "user_user = user_user.apply(lambda col: col.apply(lambda elem: str(elem).replace(',', '.'))).astype(float)\n",
    "item_item = item_item.apply(lambda col: col.apply(lambda elem: str(elem).replace(',', '.'))).astype(float)\n",
    "matrix_fact = matrix_fact.apply(lambda col: col.apply(lambda elem: str(elem).replace(',', '.'))).astype(float)\n",
    "pers_bias = pers_bias.apply(lambda col: col.apply(lambda elem: str(elem).replace(',', '.'))).astype(float)\n",
    "\n",
    "print('items.shape = ' + str(items.shape))\n",
    "print('actual_ratings.shape = ' + str(actual_ratings.shape))\n",
    "print('content_based.shape = ' + str(content_based.shape))\n",
    "print('user_user.shape = ' + str(user_user.shape))\n",
    "print('item_item.shape = ' + str(item_item.shape))\n",
    "print('matrix_fact.shape = ' + str(matrix_fact.shape))\n",
    "print('pers_bias.shape = ' + str(pers_bias.shape))\n",
    "\n",
    "actual_ratings.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Class RecommenderEvaluator\n",
    "\n",
    "In order to become easier to evaluate the metrics, I created a class that receives all the original ratings and predicted ratings for every recommender system and defined functions to extract all the metrics established in section 1 of the capstone report. Lets take a look at a summary of the class before looking at the code:\n",
    "- **Constructor (init)**: receive all recommendation algorithms, besides the actual rating list and the list of items. All data is contained in the data downloaded from Coursera. Besides storing all recommendation algorithms, the constructor also calculate the 20 most frequent items, which is used in the popularity metric calculation.\n",
    "\n",
    "- **get_observed_ratings**: as the ratings matrix is sparse, this method only returns the items a user with id userId has purchased.\n",
    "\n",
    "- **get_top_n**: by ordering all the predicted ratings for each recommendation algorithm, we can extract what would be their 'top' recommendation for a given user. Given a parameter $n$, we can then return all the top $n$ recommendations for all the recommendation algorithms.\n",
    "\n",
    "- **rmse**: by comparing the observed ratings a given user has given to an item and the predicted rating an algorithm has defined for a user, we can have an idea of how much error the algorithm is predicting the user's ratings. Here we don't work with lists, as usually each user has rated only a few amount of items. So here we get all the items the user has rated, recover these items from the algorithms' recommendations and them calculate the error.\n",
    "\n",
    "- **nDCG**: By looking at lists now, we can have an idea of how optimal the ranked lists are. By using the scoring factor defined in the report, we can calculate the overall DCG for the recommenders' lists and then normalise them using the concepts of the nDCG.\n",
    "\n",
    "- **Price and avalaibility diversity**: Diversity metric which evaluate how the recommended items' prices vary, *i.e.*, how is the standard deviation of the price. The higher, the better in this case. The same is for the availability index, but here, with higher standard deviations, it means the models are recommending items which are present and not present in local stores.\n",
    "\n",
    "- **Popularity**: A popular recommender tries to recommend items which has a high chance of being purchased. In the formulation of this metric, an item has a high chance of being purchased if lots of people have purchased them. In the class constructor, we take the observed ratings data and the item list and select which were the top $n$ (standard = 20) most purchased data. In a recommendation list, we return the ration of how many items were inside this list of top $n$ ones."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "class RecommenderEvaluator:\n",
    "    \n",
    "    def __init__(self, items, actual_ratings, content_based, user_user, item_item, matrix_fact, pers_bias):\n",
    "        \n",
    "        self.items = items\n",
    "        self.actual_ratings = actual_ratings\n",
    "        # static data containing the average score given by each user\n",
    "        self.average_rating_per_userid = actual_ratings.apply(lambda row: np.average(row[~np.isnan(row)]))\n",
    "        \n",
    "        self.content_based = content_based\n",
    "        self.user_user = user_user\n",
    "        self.item_item = item_item\n",
    "        self.matrix_fact = matrix_fact\n",
    "        self.pers_bias = pers_bias\n",
    "        \n",
    "        # aggregate list. Makes for loops among all recommenders' predictions easier\n",
    "        self.recommenders_list = [self.content_based, self.user_user, self.item_item, self.matrix_fact,self.pers_bias]\n",
    "        self.recommenders_list_names = ['content_based', 'user_user', 'item_item', 'matrix_fact','pers_bias']\n",
    "        \n",
    "        # Used for item popularity metric.\n",
    "        # Calculate the 20 most popular items (item which most of the customers bought)\n",
    "        N_LIM = 20\n",
    "        perc_users_bought_item = self.actual_ratings.apply(lambda item: np.sum(~np.isnan(item)), axis=0)/actual_ratings.shape[1]\n",
    "        sort_pop_items = np.argsort(perc_users_bought_item)[::-1]\n",
    "        self.pop_items = perc_users_bought_item.iloc[sort_pop_items][:N_LIM].index.values.astype(np.int)\n",
    "    \n",
    "        \n",
    "    def get_observed_ratings(self, userId):\n",
    "        \"\"\"\n",
    "        Returns all the items a given user evaluated and their ratings. Used mainly by all the metrics calculation\n",
    "        :parameter: userId - user id\n",
    "        :return: array of rated items. Index is the item id and value is the item rating\n",
    "        \"\"\"\n",
    "        userId = str(userId)\n",
    "        filtered_ratings = self.actual_ratings[userId]\n",
    "        rated_items = filtered_ratings[~np.isnan(filtered_ratings)]\n",
    "        return rated_items\n",
    "    \n",
    "    def get_top_n(self, userId, n):\n",
    "        \"\"\"\n",
    "        Get the top n recommendations for every recommender in the list given a user id\n",
    "        :parameter: userId - user id\n",
    "        :parameter: n - max number of recommendations to return\n",
    "        :return: dictionary where the key is the recommender's name and the value is an array of size n for the top n recommnendations.\n",
    "        \"\"\"\n",
    "        userId = str(userId)\n",
    "        predicted_ratings = dict()\n",
    "        for recommender, recommender_name in zip(self.recommenders_list,self.recommenders_list_names):\n",
    "            item_ids = recommender[userId].argsort().sort_values()[:n].index.values\n",
    "            predicted_ratings[recommender_name] = item_ids\n",
    "        return predicted_ratings\n",
    "    \n",
    "    def rmse(self, userId):\n",
    "        \"\"\"\n",
    "        Root Mean Square Error of the predicted and observed values between the recommender's prediction and the actual ratings\n",
    "        :parameter: userId - user id\n",
    "        :return: dataframe of containing the rmse from all recommenders given user id\n",
    "        \"\"\"\n",
    "        userId = str(userId)\n",
    "        observed_ratings = self.get_observed_ratings(userId)\n",
    "        rmse_list = {'rmse': []}\n",
    "        for recommender in self.recommenders_list:\n",
    "            predicted_ratings = recommender.loc[observed_ratings.index, userId]\n",
    "            rmse_list['rmse'].append(np.sqrt(np.average((predicted_ratings - observed_ratings)**2)))\n",
    "        rmse_list = pd.DataFrame(rmse_list, index = self.recommenders_list_names)\n",
    "        return rmse_list\n",
    "    \n",
    "    def nDCG(self, userId, top_n = 5, individual_recommendation = None):\n",
    "        \"\"\"\n",
    "        Normalised Discounted Cumulative Gain for all recommenders given user id\n",
    "        :parameter: userId - user id\n",
    "        :return: dataframe of containing the nDCG from all recommenders given user id\n",
    "        \"\"\"\n",
    "        ri = self.get_observed_ratings(userId)\n",
    "        \n",
    "        if(individual_recommendation is None):\n",
    "            topn = self.get_top_n(userId,top_n)\n",
    "            results_pandas_index = self.recommenders_list_names\n",
    "        else:\n",
    "            topn = individual_recommendation\n",
    "            results_pandas_index = list(individual_recommendation.keys())\n",
    "\n",
    "        # 1st step: Given recommendations, transform list into scores (see score transcriptions in the capstone report)\n",
    "        scores_all = []\n",
    "        for name, item_list in topn.items():\n",
    "            scores = np.empty_like(item_list) # initialise 'random' array\n",
    "            scores[:] = -10                   ###########################\n",
    "                                                                   # check which items returned by the recommender\n",
    "            is_already_rated = np.isin(item_list, ri.index.values) # the user already rated. Items users didn't rate\n",
    "            scores[~is_already_rated] = 0                          # receive score = 0\n",
    "            for index, score in enumerate(scores):\n",
    "                if(score != 0):                                    # for each recommended items the user rated\n",
    "                    if(ri[item_list[index]] < self.average_rating_per_userid[userId] - 1): # score accordingly the report \n",
    "                        scores[index] = -1\n",
    "                    elif((ri[item_list[index]] >= self.average_rating_per_userid[userId] - 1) & \n",
    "                         (ri[item_list[index]] < self.average_rating_per_userid[userId] + 0.5)):\n",
    "                        scores[index] = 1\n",
    "                    else:\n",
    "                        scores[index] = 2\n",
    "            scores_all.append(scores)                              # append all the transformed scores\n",
    "        scores_all  \n",
    "\n",
    "        # 2nd step: Given scores, calculate the model's DCG, ideal DCG and then nDCG\n",
    "        nDCG_all = dict()\n",
    "        \n",
    "        for index_model, scores_model in enumerate(scores_all):   # for each model\n",
    "            model_DCG = 0                                         # calculate model's DCG\n",
    "            for index, score in enumerate(scores_model):          #\n",
    "                index_ = index + 1                                #\n",
    "                model_DCG = model_DCG + score/np.log2(index_ + 1) #   \n",
    "            ideal_rank_items = np.sort(scores_model)[::-1]                        # calculate model's ideal DCG\n",
    "            ideal_rank_DCG = 0                                                    #\n",
    "            for index, ideal_score in enumerate(ideal_rank_items):                #\n",
    "                index_ = index + 1                                                #\n",
    "                ideal_rank_DCG = ideal_rank_DCG + ideal_score/np.log2(index_ + 1) #\n",
    "            if((ideal_rank_DCG == 0) | (np.abs(ideal_rank_DCG) < np.abs(model_DCG))): # if nDCG is 0 or only negative scores came up\n",
    "                nDCG = 0 \n",
    "            else:                                                     # calculate final nDCG when ideal DCG is != 0\n",
    "                nDCG = model_DCG/ideal_rank_DCG\n",
    "                                                         \n",
    "            nDCG_all[results_pandas_index[index_model]] = nDCG # save each model's nDCG in a dict\n",
    "            # convert it to dataframe\n",
    "            result_final = pd.DataFrame(nDCG_all, index=range(1)).transpose()\n",
    "            result_final.columns = ['nDCG']\n",
    "        return result_final\n",
    "\n",
    "    def price_diversity(self,userId,top_n = 5,individual_recommendation = None):\n",
    "        \"\"\"\n",
    "        Mean and standard deviation of the price of the top n products recommended by each algorithm. \n",
    "        Intuition for a high price wise diversity recommender is to have a high price standard deviation\n",
    "        :parameter: userId - user id\n",
    "        :return: dataframe of containing the price's mean and standard deviation from all recommenders given user id\n",
    "        \"\"\"\n",
    "\n",
    "        if(individual_recommendation is None):\n",
    "            topn = self.get_top_n(userId,top_n)\n",
    "        else:\n",
    "            topn = individual_recommendation\n",
    "\n",
    "        stats = pd.DataFrame()\n",
    "        for key, value in topn.items():\n",
    "            data_filtered = self.items.loc[topn[key]][['Price']].agg(['mean','std']).transpose()\n",
    "            data_filtered.index = [key]\n",
    "            stats = stats.append(data_filtered)\n",
    "        return stats\n",
    "    \n",
    "    def availability_diversity(self,userId,top_n = 5,individual_recommendation = None):\n",
    "        \"\"\"\n",
    "        Mean and standard deviation of the availabity index of the top n products recommended by each algorithm. \n",
    "        Intuition for a high availabity diversity is to have a small mean value in the availabity index\n",
    "        :parameter: userId - user id\n",
    "        :return: dataframe of containing the availabity index's mean and standard deviation from all recommenders given user id\n",
    "        \"\"\"\n",
    "        if(individual_recommendation is None):\n",
    "            topn = self.get_top_n(userId,top_n)\n",
    "        else:\n",
    "            topn = individual_recommendation\n",
    "\n",
    "        stats = pd.DataFrame()\n",
    "        for key, value in topn.items():\n",
    "            data_filtered = self.items.loc[topn[key]][['Availability']].agg(['mean','std']).transpose()\n",
    "            data_filtered.index = [key]\n",
    "            stats = stats.append(data_filtered)\n",
    "        return stats\n",
    "    \n",
    "    def popularity(self, userId,top_n = 5,individual_recommendation = None):\n",
    "        \"\"\"\n",
    "        Return the ratio of how many items of the top n items are among the most popular purchased items. Default is\n",
    "        the 20 most purchased items.\n",
    "        :parameter: userId - user id\n",
    "        :return: dataframe of containing ratio of popular items in the recommended list from all recommenders given user id\n",
    "        \"\"\"\n",
    "        if(individual_recommendation is None):\n",
    "            topn = self.get_top_n(userId,top_n)\n",
    "            results_pandas_index = self.recommenders_list_names\n",
    "        else:\n",
    "            topn = individual_recommendation\n",
    "            results_pandas_index = list(individual_recommendation.keys())\n",
    "\n",
    "        results = {'popularity': []}\n",
    "        for recommender, recommendations in topn.items():\n",
    "            popularity = np.sum(np.isin(recommendations,self.pop_items))\n",
    "            results['popularity'].append(popularity)\n",
    "        return pd.DataFrame(results,index = results_pandas_index)\n",
    "    \n",
    "    def precision_at_n(self, userId, top_n = 5, individual_recommendation = None):\n",
    "        \n",
    "        if(individual_recommendation is None):\n",
    "            topn = self.get_top_n(userId,top_n)\n",
    "            results_pandas_index = self.recommenders_list_names\n",
    "        else:\n",
    "            topn = individual_recommendation\n",
    "            results_pandas_index = list(individual_recommendation.keys())\n",
    "        \n",
    "        observed_ratings = self.get_observed_ratings(userId).index.values\n",
    "        precisions = {'precision_at_'+str(top_n): []}\n",
    "        for recommender, recommendations in topn.items():\n",
    "            precisions['precision_at_'+str(top_n)].append(np.sum(np.isin(recommendations, observed_ratings))/top_n)\n",
    "        return pd.DataFrame(precisions,index = results_pandas_index)\n",
    "    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Test methods:\n",
    "\n",
    "Just to have an idea of the output of each method, lets call all them with a test user. At the next section we will calculate these metrics for all users."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "userId = '64'\n",
    "re = RecommenderEvaluator(items, actual_ratings, content_based, user_user, item_item, matrix_fact, pers_bias)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Test RMSE"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>rmse</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>content_based</th>\n",
       "      <td>0.772809</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>user_user</th>\n",
       "      <td>0.624086</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>item_item</th>\n",
       "      <td>0.797922</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>matrix_fact</th>\n",
       "      <td>0.853212</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>pers_bias</th>\n",
       "      <td>0.845591</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                   rmse\n",
       "content_based  0.772809\n",
       "user_user      0.624086\n",
       "item_item      0.797922\n",
       "matrix_fact    0.853212\n",
       "pers_bias      0.845591"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "re.rmse(userId)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Test nDCG"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>nDCG</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>content_based</th>\n",
       "      <td>0.00000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>item_item</th>\n",
       "      <td>0.63093</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>matrix_fact</th>\n",
       "      <td>0.00000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>pers_bias</th>\n",
       "      <td>0.00000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>user_user</th>\n",
       "      <td>0.00000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                  nDCG\n",
       "content_based  0.00000\n",
       "item_item      0.63093\n",
       "matrix_fact    0.00000\n",
       "pers_bias      0.00000\n",
       "user_user      0.00000"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "re.nDCG(userId)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Test Diversity - Price and Availability"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>mean</th>\n",
       "      <th>std</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>content_based</th>\n",
       "      <td>10.376</td>\n",
       "      <td>5.160923</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>user_user</th>\n",
       "      <td>19.846</td>\n",
       "      <td>14.888584</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>item_item</th>\n",
       "      <td>6.518</td>\n",
       "      <td>3.736117</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>matrix_fact</th>\n",
       "      <td>9.706</td>\n",
       "      <td>5.622004</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>pers_bias</th>\n",
       "      <td>9.890</td>\n",
       "      <td>5.121875</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                 mean        std\n",
       "content_based  10.376   5.160923\n",
       "user_user      19.846  14.888584\n",
       "item_item       6.518   3.736117\n",
       "matrix_fact     9.706   5.622004\n",
       "pers_bias       9.890   5.121875"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "re.price_diversity(userId)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>mean</th>\n",
       "      <th>std</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>content_based</th>\n",
       "      <td>0.595712</td>\n",
       "      <td>0.197574</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>user_user</th>\n",
       "      <td>0.628495</td>\n",
       "      <td>0.124328</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>item_item</th>\n",
       "      <td>0.531987</td>\n",
       "      <td>0.228363</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>matrix_fact</th>\n",
       "      <td>0.588537</td>\n",
       "      <td>0.197005</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>pers_bias</th>\n",
       "      <td>0.588596</td>\n",
       "      <td>0.172630</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                   mean       std\n",
       "content_based  0.595712  0.197574\n",
       "user_user      0.628495  0.124328\n",
       "item_item      0.531987  0.228363\n",
       "matrix_fact    0.588537  0.197005\n",
       "pers_bias      0.588596  0.172630"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "re.availability_diversity(userId)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Test Popularity"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>popularity</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>content_based</th>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>user_user</th>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>item_item</th>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>matrix_fact</th>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>pers_bias</th>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "               popularity\n",
       "content_based           0\n",
       "user_user               0\n",
       "item_item               0\n",
       "matrix_fact             0\n",
       "pers_bias               0"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "re.popularity(userId)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Test Precision@N"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>precision_at_5</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>content_based</th>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>user_user</th>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>item_item</th>\n",
       "      <td>0.2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>matrix_fact</th>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>pers_bias</th>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "               precision_at_5\n",
       "content_based             0.0\n",
       "user_user                 0.0\n",
       "item_item                 0.2\n",
       "matrix_fact               0.0\n",
       "pers_bias                 0.0"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "re.precision_at_n(userId)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Average metrics by all users\n",
    "\n",
    "Espefically for user 907, the recommendations from the user user came with all nulls (original dataset). This specifically impacted the RMSE calculation, as one Nan damaged the entire average calculation. So specifically for RMSE we did a separate calculation section. All the other metrics are going the be calculated in the next code block."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Average RMSE for all users\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "content_based    0.572387\n",
       "user_user        0.545130\n",
       "item_item        0.574672\n",
       "matrix_fact      0.659029\n",
       "pers_bias        0.666273\n",
       "Name: rmse, dtype: float64"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "re = RecommenderEvaluator(items, actual_ratings, content_based, user_user, item_item, matrix_fact, pers_bias)\n",
    "\n",
    "i = 0\n",
    "count = np.array([0,0,0,0,0])\n",
    "for userId in actual_ratings.columns:\n",
    "    if(userId == '907'):\n",
    "        rmse_recommenders = re.rmse(userId).fillna(0)\n",
    "    else:\n",
    "        rmse_recommenders = re.rmse(userId)\n",
    "    count = count + rmse_recommenders['rmse']\n",
    "\n",
    "# as we didn't use user 907 for user user, divide it by the number of users - 1\n",
    "denominator = [len(actual_ratings.columns)] * 5\n",
    "denominator[1] = len(actual_ratings.columns) - 1\n",
    "print('Average RMSE for all users')\n",
    "count/ denominator"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "---\n",
      "Average nDCG\n",
      "---\n",
      "\n",
      "content_based    0.136505\n",
      "item_item        0.146798\n",
      "matrix_fact      0.155888\n",
      "pers_bias        0.125180\n",
      "user_user        0.169080\n",
      "Name: nDCG, dtype: float64\n",
      "\n",
      "---\n",
      "Average Price - Diversity Measure\n",
      "---\n",
      "\n",
      "                    mean        std\n",
      "content_based  19.286627  19.229536\n",
      "user_user      21.961776  25.275120\n",
      "item_item      25.931943  32.224609\n",
      "matrix_fact    21.165554  26.236822\n",
      "pers_bias       9.938984   5.159261\n",
      "\n",
      "---\n",
      "Average Availability - Diversity Measure\n",
      "---\n",
      "\n",
      "                   mean       std\n",
      "content_based  0.623888  0.225789\n",
      "user_user      0.682751  0.230219\n",
      "item_item      0.655725  0.223781\n",
      "matrix_fact    0.601153  0.202596\n",
      "pers_bias      0.638596  0.202630\n",
      "\n",
      "---\n",
      "Average Popularity\n",
      "---\n",
      "\n",
      "content_based    0.00\n",
      "user_user        0.01\n",
      "item_item        0.00\n",
      "matrix_fact      0.00\n",
      "pers_bias        0.00\n",
      "Name: popularity, dtype: float64\n",
      "---\n",
      "\n",
      "Average Precision@5\n",
      "---\n",
      "\n",
      "content_based    0.050\n",
      "user_user        0.066\n",
      "item_item        0.076\n",
      "matrix_fact      0.064\n",
      "pers_bias        0.052\n",
      "Name: precision_at_5, dtype: float64\n"
     ]
    }
   ],
   "source": [
    "count_nDCG = np.array([0,0,0,0,0])\n",
    "count_diversity_price = np.ndarray([5,2])\n",
    "count_diversity_availability = np.ndarray([5,2])\n",
    "count_popularity = np.array([0,0,0,0,0])\n",
    "count_precision_at_5 = np.array([0,0,0,0,0])\n",
    "\n",
    "for userId in actual_ratings.columns:\n",
    "    nDCG_recommenders = re.nDCG(userId)\n",
    "    count_nDCG = count_nDCG + nDCG_recommenders['nDCG']\n",
    "    \n",
    "    diversity_price_recommenders = re.price_diversity(userId)\n",
    "    count_diversity_price = count_diversity_price + diversity_price_recommenders[['mean','std']]\n",
    "    \n",
    "    diversity_availability_recommenders = re.availability_diversity(userId)\n",
    "    count_diversity_availability = count_diversity_availability + diversity_availability_recommenders[['mean','std']]\n",
    "    \n",
    "    popularity_recommenders = re.popularity(userId)\n",
    "    count_popularity = count_popularity + popularity_recommenders['popularity'] \n",
    "    \n",
    "    precision_recommenders = re.precision_at_n(userId)\n",
    "    count_precision_at_5 = count_precision_at_5 + precision_recommenders['precision_at_5'] \n",
    "\n",
    "print('\\n---')\n",
    "print('Average nDCG')\n",
    "print('---\\n')\n",
    "print(count_nDCG/len(actual_ratings.columns))\n",
    "print('\\n---')\n",
    "print('Average Price - Diversity Measure')\n",
    "print('---\\n')\n",
    "print(count_diversity_price/len(actual_ratings.columns))\n",
    "print('\\n---')\n",
    "print('Average Availability - Diversity Measure')\n",
    "print('---\\n')\n",
    "print(count_diversity_availability/len(actual_ratings.columns))\n",
    "print('\\n---')\n",
    "print('Average Popularity')\n",
    "print('---\\n')\n",
    "print(count_popularity/len(actual_ratings.columns))\n",
    "print('---\\n')\n",
    "print('Average Precision@5')\n",
    "print('---\\n')\n",
    "print(count_precision_at_5/len(actual_ratings.columns))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Final Analysis\n",
    "\n",
    "In terms of **RMSE**, the user-user collaborative filtering showed to be the most effective, despite it not being significantly better.\n",
    "\n",
    "For nDCG rank score, again user user and now item item collaborative filtering were the best.\n",
    "\n",
    "In terms of price diversity, the item item algorith was the most diverse, providing products varying ~32 dollars from the mean item price list. Matrix factorisation and user user follow right behind, with price standard deviation around 25 dollars. An interesting factor here was the *pers_bias* algorithm, as it recommended basically cheap products with a low standard deviation.\n",
    "\n",
    "For the availabity index, all the algorithms besides the user user managed to recommend items not so present in the local stores **together** with items present in local stores, as we can see they also provided items with availability index high (high standard deviation).\n",
    "\n",
    "In terms of popularity, no algorithm actually managed to obtain good scores in the way we defined. So, if the popularity is focused in the future, we can either change the popularity concept or improve mechanics in the recommender so it predict higher scores for the most popular items in the store.\n",
    "\n",
    "After this evaluation, it seemed to us that the item-item recommender system had an overall better performance, highlighted in terms of its diversity scores. Unfortunately, the items that item item recommender has suggested are in overall pricy, and we can check if there is any mixture possibility with the pers_bias algorithm, as it really indicated cheap prices and a low price standard deviation. Matrix factorization performed good as well but it didn't outperform any of the other recommenders."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Hibridization Techniques - Part III\n",
    "\n",
    "We are trying four different types of hibridization here.\n",
    "\n",
    "1. Linear ensemble\n",
    "2. Non linear ensemble\n",
    "3. Top 1 from each recommender\n",
    "4. Recommender switching\n",
    "  \n",
    "The first two options approach the recommender's performance in terms of how good it predicts the users' ratings, so its only evaluation will be in terms of RMSE.\n",
    "  \n",
    "The third approach have the intuition that, if we get the top 1 recommendation from each algorithm, the resulting 5 item list will have a better performance in terms of identyfing 'good' items to users. In this case, we defined the good items if the recommender suggested an already bought item for an user. Therefore, the final measurement of this hibridization mechanism is through the precision@5, as we end up with a 5 item list.\n",
    "\n",
    "The final mixing algorithm has the underlying theory of how collaborative filtering mechanisms perform with items that had not enough users/items in its calculations. As a well known weakness of these recommenders, the idea was to check how many items we would affect if we established a threshold of enough data in order for us to use a collaborative filtering. Otherwise, if the item doesn't have enough support in form of users' ratings we could have a support of a content based recommendation, or even, in last case, a non personalised one.\n",
    "  \n",
    "  \n",
    "## Dataset Creation and User Sample Definition\n",
    "\n",
    "### Dataset\n",
    "\n",
    "For the first and second approach, we need another perspective on the data. The dataset contains all the existing ratings from all users and concatenates all the predictions made the 5 traditional recommenders. The idea is to use the observed rating as target variable and all recommenders' predictions as dependent variable, *i.e.* treat this as a regression problems."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>content_based</th>\n",
       "      <th>item_item</th>\n",
       "      <th>matrix_fact</th>\n",
       "      <th>pers_bias</th>\n",
       "      <th>rating</th>\n",
       "      <th>user_user</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>4.204670</td>\n",
       "      <td>4.020528</td>\n",
       "      <td>4.293189</td>\n",
       "      <td>4.221597</td>\n",
       "      <td>4.0</td>\n",
       "      <td>3.997032</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>4.784942</td>\n",
       "      <td>4.850921</td>\n",
       "      <td>4.703951</td>\n",
       "      <td>4.659690</td>\n",
       "      <td>5.0</td>\n",
       "      <td>4.962889</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>4.522897</td>\n",
       "      <td>4.538863</td>\n",
       "      <td>4.653262</td>\n",
       "      <td>4.613524</td>\n",
       "      <td>4.0</td>\n",
       "      <td>4.111147</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4.721176</td>\n",
       "      <td>4.687212</td>\n",
       "      <td>4.675896</td>\n",
       "      <td>4.657269</td>\n",
       "      <td>5.0</td>\n",
       "      <td>5.040519</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4.368981</td>\n",
       "      <td>4.321261</td>\n",
       "      <td>4.358705</td>\n",
       "      <td>4.352745</td>\n",
       "      <td>3.0</td>\n",
       "      <td>4.460465</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   content_based  item_item  matrix_fact  pers_bias  rating  user_user\n",
       "0       4.204670   4.020528     4.293189   4.221597     4.0   3.997032\n",
       "1       4.784942   4.850921     4.703951   4.659690     5.0   4.962889\n",
       "2       4.522897   4.538863     4.653262   4.613524     4.0   4.111147\n",
       "3       4.721176   4.687212     4.675896   4.657269     5.0   5.040519\n",
       "4       4.368981   4.321261     4.358705   4.352745     3.0   4.460465"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "obs_ratings_list = []\n",
    "content_based_list = []\n",
    "user_user_list = []\n",
    "item_item_list = []\n",
    "matrix_fact_list = []\n",
    "pers_bias_list = []\n",
    "\n",
    "re = RecommenderEvaluator(items, actual_ratings, content_based, user_user, item_item, matrix_fact, pers_bias)\n",
    "for userId in actual_ratings.columns:\n",
    "    \n",
    "    observed_ratings = re.get_observed_ratings(userId)\n",
    "    obs_ratings_list.extend(observed_ratings.values)\n",
    "    \n",
    "    content_based_list.extend(content_based.loc[observed_ratings.index, userId].values)\n",
    "    user_user_list.extend(user_user.loc[observed_ratings.index, userId].values)\n",
    "    item_item_list.extend(item_item.loc[observed_ratings.index, userId].values)\n",
    "    matrix_fact_list.extend(matrix_fact.loc[observed_ratings.index, userId].values)\n",
    "    pers_bias_list.extend(pers_bias.loc[observed_ratings.index, userId].values)\n",
    "dataset = pd.DataFrame({'rating': obs_ratings_list, 'content_based':content_based_list, 'user_user': user_user_list,\n",
    "             'item_item':item_item_list, 'matrix_fact':matrix_fact_list,'pers_bias':pers_bias_list})\n",
    "dataset = dataset.dropna()\n",
    "dataset.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### In order to have an idea of the results, let's choose 3 users randomly to show the predictions using the new hybrid models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "sample_users: ['1528' '3524' '417']\n"
     ]
    }
   ],
   "source": [
    "np.random.seed(42)\n",
    "sample_users = np.random.choice(actual_ratings.columns, 3).astype(str)\n",
    "print('sample_users: ' + str(sample_users))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Get recommenders' predictions  for sample users in order to create input for ensemble models (hybridization I and II)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user</th>\n",
       "      <th>item</th>\n",
       "      <th>content_based</th>\n",
       "      <th>user_user</th>\n",
       "      <th>item_item</th>\n",
       "      <th>matrix_fact</th>\n",
       "      <th>pers_bias</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1528</td>\n",
       "      <td>24</td>\n",
       "      <td>5.041995</td>\n",
       "      <td>4.550045</td>\n",
       "      <td>4.847393</td>\n",
       "      <td>4.957907</td>\n",
       "      <td>4.934653</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1528</td>\n",
       "      <td>30</td>\n",
       "      <td>4.838702</td>\n",
       "      <td>4.700607</td>\n",
       "      <td>4.435800</td>\n",
       "      <td>4.944114</td>\n",
       "      <td>4.936491</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1528</td>\n",
       "      <td>35</td>\n",
       "      <td>4.678181</td>\n",
       "      <td>4.683325</td>\n",
       "      <td>4.683956</td>\n",
       "      <td>4.799281</td>\n",
       "      <td>4.773595</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1528</td>\n",
       "      <td>41</td>\n",
       "      <td>4.851023</td>\n",
       "      <td>4.649232</td>\n",
       "      <td>4.396546</td>\n",
       "      <td>4.972493</td>\n",
       "      <td>4.944475</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1528</td>\n",
       "      <td>45</td>\n",
       "      <td>5.013797</td>\n",
       "      <td>4.963995</td>\n",
       "      <td>4.914717</td>\n",
       "      <td>4.953208</td>\n",
       "      <td>4.932321</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   user  item  content_based  user_user  item_item  matrix_fact  pers_bias\n",
       "0  1528    24       5.041995   4.550045   4.847393     4.957907   4.934653\n",
       "1  1528    30       4.838702   4.700607   4.435800     4.944114   4.936491\n",
       "2  1528    35       4.678181   4.683325   4.683956     4.799281   4.773595\n",
       "3  1528    41       4.851023   4.649232   4.396546     4.972493   4.944475\n",
       "4  1528    45       5.013797   4.963995   4.914717     4.953208   4.932321"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from collections import OrderedDict\n",
    "\n",
    "df_sample = pd.DataFrame()\n",
    "for user in sample_users:\n",
    "    content_based_ = re.content_based[user]\n",
    "    user_user_ = re.user_user[user]\n",
    "    item_item_ = re.item_item[user]\n",
    "    matrix_fact_ = re.matrix_fact[user]\n",
    "    pers_bias_ = re.pers_bias[user]\n",
    "    df_sample = df_sample.append(pd.DataFrame(OrderedDict({'user':user,'item':actual_ratings.index.values,'content_based':content_based_, 'user_user':user_user_, 'item_item':item_item_,\n",
    "              'matrix_fact':matrix_fact_,'pers_bias':pers_bias_})), ignore_index=True)\n",
    "    \n",
    "df_sample.head()\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "\n",
    "## Focus on Performance (RMSE) I - Linear Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "RMSE for linear ensemble of recommender systems:\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0.6250427976675703"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.linear_model import LinearRegression\n",
    "from sklearn.model_selection import cross_val_score\n",
    "\n",
    "linear = LinearRegression()\n",
    "print('RMSE for linear ensemble of recommender systems:')\n",
    "np.mean(cross_val_score(linear, dataset.drop('rating', axis=1), dataset['rating'], cv=5))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Predictions for sample users: Creating top 5 recommendations for sample users"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>user</th>\n",
       "      <th>item</th>\n",
       "      <th>predictions</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>user</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"5\" valign=\"top\">1528</th>\n",
       "      <th>25</th>\n",
       "      <td>1528</td>\n",
       "      <td>454</td>\n",
       "      <td>5.168855</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>187</th>\n",
       "      <td>1528</td>\n",
       "      <td>2257</td>\n",
       "      <td>5.129061</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>88</th>\n",
       "      <td>1528</td>\n",
       "      <td>1412</td>\n",
       "      <td>5.066383</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>1528</td>\n",
       "      <td>517</td>\n",
       "      <td>5.045874</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>184</th>\n",
       "      <td>1528</td>\n",
       "      <td>2247</td>\n",
       "      <td>5.036847</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"5\" valign=\"top\">3524</th>\n",
       "      <th>246</th>\n",
       "      <td>3524</td>\n",
       "      <td>1032</td>\n",
       "      <td>5.119736</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>304</th>\n",
       "      <td>3524</td>\n",
       "      <td>1518</td>\n",
       "      <td>5.116655</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>285</th>\n",
       "      <td>3524</td>\n",
       "      <td>1387</td>\n",
       "      <td>5.008854</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>326</th>\n",
       "      <td>3524</td>\n",
       "      <td>1798</td>\n",
       "      <td>4.967117</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>375</th>\n",
       "      <td>3524</td>\n",
       "      <td>2222</td>\n",
       "      <td>4.965883</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"5\" valign=\"top\">417</th>\n",
       "      <th>427</th>\n",
       "      <td>417</td>\n",
       "      <td>517</td>\n",
       "      <td>4.530780</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>466</th>\n",
       "      <td>417</td>\n",
       "      <td>1297</td>\n",
       "      <td>4.506569</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>493</th>\n",
       "      <td>417</td>\n",
       "      <td>1455</td>\n",
       "      <td>4.473096</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>552</th>\n",
       "      <td>417</td>\n",
       "      <td>2025</td>\n",
       "      <td>4.469837</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>495</th>\n",
       "      <td>417</td>\n",
       "      <td>1461</td>\n",
       "      <td>4.452051</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          user  item  predictions\n",
       "user                             \n",
       "1528 25   1528   454     5.168855\n",
       "     187  1528  2257     5.129061\n",
       "     88   1528  1412     5.066383\n",
       "     27   1528   517     5.045874\n",
       "     184  1528  2247     5.036847\n",
       "3524 246  3524  1032     5.119736\n",
       "     304  3524  1518     5.116655\n",
       "     285  3524  1387     5.008854\n",
       "     326  3524  1798     4.967117\n",
       "     375  3524  2222     4.965883\n",
       "417  427   417   517     4.530780\n",
       "     466   417  1297     4.506569\n",
       "     493   417  1455     4.473096\n",
       "     552   417  2025     4.469837\n",
       "     495   417  1461     4.452051"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pred_cols = ['content_based','user_user','item_item','matrix_fact','pers_bias']\n",
    "predictions = linear.fit(dataset.drop('rating', axis=1), dataset['rating']).predict(df_sample[pred_cols])\n",
    "recommendations = pd.DataFrame(OrderedDict({'user':df_sample['user'], 'item':df_sample['item'], 'predictions':predictions}))\n",
    "recommendations.groupby('user').apply(lambda df_user : df_user.loc[df_user['predictions'].sort_values(ascending=False)[:5].index.values])\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Focus on Performance (RMSE) II - Emsemble"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "RMSE for non linear ensemble of recommender systems:\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0.5708441816222726"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.ensemble import RandomForestRegressor\n",
    "\n",
    "rf = RandomForestRegressor(random_state=42)\n",
    "print('RMSE for non linear ensemble of recommender systems:')\n",
    "np.mean(cross_val_score(rf, dataset.drop('rating', axis=1), dataset['rating'], cv=5))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Predictions for sample users:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>user</th>\n",
       "      <th>item</th>\n",
       "      <th>predictions</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>user</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"5\" valign=\"top\">1528</th>\n",
       "      <th>199</th>\n",
       "      <td>1528</td>\n",
       "      <td>2394</td>\n",
       "      <td>5.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>188</th>\n",
       "      <td>1528</td>\n",
       "      <td>2258</td>\n",
       "      <td>5.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>66</th>\n",
       "      <td>1528</td>\n",
       "      <td>1297</td>\n",
       "      <td>5.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>67</th>\n",
       "      <td>1528</td>\n",
       "      <td>1298</td>\n",
       "      <td>5.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>93</th>\n",
       "      <td>1528</td>\n",
       "      <td>1455</td>\n",
       "      <td>5.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"5\" valign=\"top\">3524</th>\n",
       "      <th>335</th>\n",
       "      <td>3524</td>\n",
       "      <td>1820</td>\n",
       "      <td>5.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>392</th>\n",
       "      <td>3524</td>\n",
       "      <td>2316</td>\n",
       "      <td>5.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>349</th>\n",
       "      <td>3524</td>\n",
       "      <td>1951</td>\n",
       "      <td>5.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>262</th>\n",
       "      <td>3524</td>\n",
       "      <td>1292</td>\n",
       "      <td>5.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>386</th>\n",
       "      <td>3524</td>\n",
       "      <td>2255</td>\n",
       "      <td>5.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"5\" valign=\"top\">417</th>\n",
       "      <th>459</th>\n",
       "      <td>417</td>\n",
       "      <td>1245</td>\n",
       "      <td>4.9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>525</th>\n",
       "      <td>417</td>\n",
       "      <td>1796</td>\n",
       "      <td>4.9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>474</th>\n",
       "      <td>417</td>\n",
       "      <td>1324</td>\n",
       "      <td>4.8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>430</th>\n",
       "      <td>417</td>\n",
       "      <td>619</td>\n",
       "      <td>4.8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>449</th>\n",
       "      <td>417</td>\n",
       "      <td>1118</td>\n",
       "      <td>4.7</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          user  item  predictions\n",
       "user                             \n",
       "1528 199  1528  2394          5.0\n",
       "     188  1528  2258          5.0\n",
       "     66   1528  1297          5.0\n",
       "     67   1528  1298          5.0\n",
       "     93   1528  1455          5.0\n",
       "3524 335  3524  1820          5.0\n",
       "     392  3524  2316          5.0\n",
       "     349  3524  1951          5.0\n",
       "     262  3524  1292          5.0\n",
       "     386  3524  2255          5.0\n",
       "417  459   417  1245          4.9\n",
       "     525   417  1796          4.9\n",
       "     474   417  1324          4.8\n",
       "     430   417   619          4.8\n",
       "     449   417  1118          4.7"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "predictions = rf.fit(dataset.drop('rating', axis=1), dataset['rating']).predict(df_sample[pred_cols])\n",
    "recommendations = pd.DataFrame(OrderedDict({'user':df_sample['user'], 'item':df_sample['item'], 'predictions':predictions}))\n",
    "recommendations.groupby('user').apply(lambda df_user : df_user.loc[df_user['predictions'].sort_values(ascending=False)[:5].index.values])\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Focus on Recommendations - Top 1 from each Recommender\n",
    "\n",
    "With the all top 1 recommender, we can evaluate its performance not just with RMSE, but all the list metrics we evaluated before. As a business constraint, we will also pay more attention to the *precision@5* metric, as a general information on how good is the recommender on providing suggestions that the user will buy, or already bought in this case.\n",
    "The majority of metrics were in the same scale as the best metrics in the all models comparison. However, it's good to highlight the the top 1 all recommender had the best *precision@5* metric among all recommender, showing to be a **good suitable hibridization mechanism**."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "---\n",
      "Average nDCG\n",
      "---\n",
      "\n",
      "top_1_all    0.159211\n",
      "Name: nDCG, dtype: float64\n",
      "\n",
      "---\n",
      "Average Price - Diversity Measure\n",
      "---\n",
      "\n",
      "              mean        std\n",
      "top_1_all  16.4625  14.741783\n",
      "\n",
      "---\n",
      "Average Availability - Diversity Measure\n",
      "---\n",
      "\n",
      "               mean       std\n",
      "top_1_all  0.575683  0.161168\n",
      "\n",
      "---\n",
      "Average Popularity\n",
      "---\n",
      "\n",
      "top_1_all    0.0\n",
      "Name: popularity, dtype: float64\n",
      "\n",
      "---\n",
      "Average Precision@5\n",
      "---\n",
      "\n",
      "top_1_all    0.082\n",
      "Name: precision_at_5, dtype: float64\n"
     ]
    }
   ],
   "source": [
    "count_nDCG = np.array([0])\n",
    "count_diversity_price = np.ndarray([1,2])\n",
    "count_diversity_availability = np.ndarray([1,2])\n",
    "count_popularity = np.array([0])\n",
    "count_precision = np.array([0])\n",
    "\n",
    "for userId in actual_ratings.columns:\n",
    "    \n",
    "    top_n_1 = re.get_top_n(userId,1)\n",
    "    user_items = {}\n",
    "    user_items['top_1_all'] = [a[0] for a in top_n_1.values()]\n",
    "    \n",
    "    nDCG_recommenders = re.nDCG(userId, individual_recommendation = user_items)\n",
    "    count_nDCG = count_nDCG + nDCG_recommenders['nDCG']\n",
    "    \n",
    "    diversity_price_recommenders = re.price_diversity(userId, individual_recommendation = user_items)\n",
    "    count_diversity_price = count_diversity_price + diversity_price_recommenders[['mean','std']]\n",
    "    \n",
    "    diversity_availability_recommenders = re.availability_diversity(userId, individual_recommendation = user_items)\n",
    "    count_diversity_availability = count_diversity_availability + diversity_availability_recommenders[['mean','std']]\n",
    "    \n",
    "    popularity_recommenders = re.popularity(userId, individual_recommendation = user_items)\n",
    "    count_popularity = count_popularity + popularity_recommenders['popularity'] \n",
    "    \n",
    "    precision_recommenders = re.precision_at_n(userId, individual_recommendation = user_items)\n",
    "    count_precision = count_precision + precision_recommenders['precision_at_5'] \n",
    "\n",
    "print('\\n---')\n",
    "print('Average nDCG')\n",
    "print('---\\n')\n",
    "print(count_nDCG/len(actual_ratings.columns))\n",
    "print('\\n---')\n",
    "print('Average Price - Diversity Measure')\n",
    "print('---\\n')\n",
    "print(count_diversity_price/len(actual_ratings.columns))\n",
    "print('\\n---')\n",
    "print('Average Availability - Diversity Measure')\n",
    "print('---\\n')\n",
    "print(count_diversity_availability/len(actual_ratings.columns))\n",
    "print('\\n---')\n",
    "print('Average Popularity')\n",
    "print('---\\n')\n",
    "print(count_popularity/len(actual_ratings.columns))\n",
    "print('\\n---')\n",
    "print('Average Precision@5')\n",
    "print('---\\n')\n",
    "print(count_precision/len(actual_ratings.columns))\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Predictions for sample users:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'1528': [2319, 1224, 2236, 2255, 2255],\n",
       " '3524': [2324, 1531, 2317, 2255, 2255],\n",
       " '417': [2236, 2258, 1943, 2258, 2255]}"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "results = {}\n",
    "for user_sample in sample_users:\n",
    "    results[user_sample] = [a[0] for a in list(re.get_top_n(user_sample, 1).values())]\n",
    "results"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##  Focus on Recommendations - Switching algorithm\n",
    "\n",
    "### Can we use a Content Based Recommender for items with less evaluations?\n",
    "\n",
    "We can see in the cumulative histogram that only around 20% of the rated items had 10 or more ratings. This signals us that maybe we can prioritize the use of a content based recommender or even a non personalised one for the majority of the items which don't have a sufficient amount of ratings in order to make the collaborative filtering algorithms to be stable."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAA3gAAADgCAYAAABRs8T9AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAAIABJREFUeJzs3Xd8VfX5wPHPk51ABiEhEFaAsLciKGBF0Tpxax21rkqH2lqtba1WqVq1ttqfW3GLdU+0TtxBUUGGQghENiQhCSMDsp/fH+cELiHjZnFyb57363VeuffM5557c855zvkOUVWMMcYYY4wxxgS+EK8DMMYYY4wxxhjTNizBM8YYY4wxxpggYQmeMcYYY4wxxgQJS/CMMcYYY4wxJkhYgmeMMcYYY4wxQcISPGOMMcYYY4wJEpbgmTYhIstFZJrXcXhJRE4TkY0iUiIi472OxysicquIFIhIbjtuo0REBrbX+o0xxh8iMktEnm3F8m1+7hSRNBFREQlrYPpfReSxttxmoBGRFBH5XESKReQur+PxiohMEZHV7jn11HbaxsMi8rf2WLdpmCV4pkkisk5Ejq4z7iIRyah9r6ojVfXTJtbT6EknCPwbuEJVu6rqYq+DaQkRmSYim1qxfF/gGmCEqvZso5g+FZFf+o5z9/Gatli/MSbwiMh5IrLQvTDNEZF3RWSq13E1RkSeEpFbfcf5c+5sa6p6m6r+sqn56jv2BpGZQAEQp6rXeB1MS9V3fdZMNwP3u+fUN9ognn2uDQFU9deqektr122axxI8EzQ6QOLYH1jucQxe6w8UqupWf2buAN+ZMSbAiMjVwP8BtwEpQD/gQeAUL+My/usAx/7+wApVVY/j8Jrf1y0d4DszzaGqNtjQ6ACsA46uM+4iIKO+eYCJwEKgCMgD7nbHbwAUKHGHw3BuMtwArAe2As8A8T7r/YU7rRD4W53tzAJeAZ51t/VLd9tfATuAHOB+IMJnfQr8FlgNFAO3AIPcZYqAl3znr/OZ640ViHQ/jwKlwI8NLH8PsNHdziLgcJ9ps4CX3c9SDHwPDAGuc7e1Efipz/ypwFxgG5ANXOYz7SngVp/304BNdb6rPwLLgJ3Ai0AU0AXYDdT4fEep9XyOePez57v74gZ33xxdZ/mn6ll2GrAJ+DOQC8wBugFvu+vb7r7u487/D6AaKHPXeb/P95ju83kfAP7n7ruvgUE+2/wpkOV+1geBz4BfutPS3fc7ce7mvuj1/5sNNtjQ8OAef0qAsxqZx59j4LXuMbAUeBwnUXzXPYbMA7rVt6zP8r7noWd9pr3sHtt2Ap8DI93xM4FKoMKN/y3fdeEc03cDiT7rGu8el8Ld95cAme5x8n2gfwOfP809Rl6Ic94tAK73mb4nZpxj/7M459gdwLfuvmjo2DvZnWen+3eyz3oHuJ+5dh8+4LOd2pgudWP6vLH95fM9Puh+LyXAfKAnTnK/HVgJjG/kd1BvrO56fb+Lo+tZ9kRgMc75eiMwq579e7E7bTvwa+AQnN/Ujtr95c7f4HUO/v2+XnKXKcZJxia40+bgnG93u5/jTw3sh8twrhO24Vw3pLrjf6yzfGQ9y67DOV8vA8qBMOAv7rLFwArgNHfe4e7vpdpd3466/4/svQa4xt0XOcDFPtvrDrzl7vdvgVtxrzUBAf7jLrfTjWmU18ekjjp4HoANHX+g+QneV8AF7uuuwKHu69qDYpjPcpe4B56B7ryvAXPcaSPcg8RUIAKnCGRlnQNfJXCqewCNBg4GDnUPQmk4J8OrfLan7gEuDhjpHrA+crcf7x6sLmxgPzQYq8+60xvZjz93D15h7sEtF4jy+SxlwLHu9GeAtcD1QDjOAXqtz7o+wznxRQHjcJKj6e60PQdT9/009r+4+QbngiLR3Ue/rm/eBj7HM8CbQKy7j1cBl/qzvDu9CvgnTmIc7e6TM4AYd50vA2/4LPMpbkJW3752P+82nOQ+DPgv8II7LQnnRHG6O+337m+mNsF73t3HIe6+nOr1/5sNNtjQ8AAc5x5DwhqZx59j4AKcRKY3zgXjdzgJVSTwMXBTfcv6LN9QgneJexyLxElEljQUVz3r+ph9b9b9C3jYfX0qzvlnuHssuwH4soHPn+YeIx91j7Fjcc51w+vGDPwK54I6BgjFOYfGudP2OfbinC+2Axe4MZzrvu/uTv8K5zwdgXPeLmL/BO8ZnJuJ0X7urwI3pih3/6zFufEbinPx/0kD+6CpWPf7LuosPw0YjXNuGINzs/rUOp/lYTeun+Kcv98AerD3N3WEz2ds6DpnGk3/vsqAE9zPfDuwoL55G/gcR7n78CB3H9+Hm1z7ufw6YAnQ1+c7Owvn+iEE+BnOTZJe7rSL8Lk2rLuv2XsNcDPOtc0JwC723lB5wR1icK4BN7I3wTsW5+Z4Ak6yN7x2uzbsP1gRTeOvN0RkR+2Ak1w0pBJIF5EkVS1R1QWNzHs+zhO+NapagvPE6hy3KMCZOHc5M1S1ArgR56Dq6ytVfUNVa1R1t6ouUtUFqlqlquuAR4Aj6izzT1UtUtXlwA/AB+72d+LcKWyogZTGYm2Sqj6rqoVubHfhHGyH+szyhaq+r6pVOElOMnCHqlbiHPDSRCTBrec2Ffizqpap6hLgMZwTmb/uVdUtqroN5+Q+zp+FRCQU54B+naoWu/v4rmZuuwbn4qnc/c4KVfVVVd2lqsU4d47rfmdNeU1Vv3H33X/Z+3lOAJar6mvutHtxEutalThFVFLdfblP3QFjTIfTHShw/59b4z5VzVPVzcAXwNequlhVy4HXafg80ChVfcI9NpbjXJyPFZF4Pxd/DicRQUQEOMcdB04idruqZrqf/TZgnIj0b2R9f3ePsUuBpTiJXl2VOPs0XVWr3XNoUQPrOxFYrapz3PPY8zhP0WaISD+cJ1g3qmqFeyydW886ZqlqqaruBr/21+tuTGU430uZqj6jqtU4pU8a+p4ajLWB+fehqp+q6vfutcUynJuBdc9Lt7jnjQ9wkpznVXWrz2+qNrZWXTvgJDjvuJ95DvV/jw05H3hCVb9z9/F1wGEiktaMddyrqht9vrOX3euHGlV9EadE1MRmrK8SuFlVK1X1HZwb+UPd64szcK4PdqnqCuDpOsvFAsMAcf8Xcpqx3U7FEjzjr1NVNaF2wCnm2JBLcYoXrhSRb0XkpEbmTcUptlBrPc7dthR32sbaCaq6C6cYia+Nvm9EZIiIvC0iuSJShHMSTKqzTJ7P6931vO/aglibJCLXiEimiOx0k+T4OrHVjaPAPaDXvseNLRXY5iZDvrH09icOl2+Ss4uGP3NdSTh3Z+vuh+ZsO989WQMgIjEi8oiIrHe/s8+BBPdg76+GPk/d35DiFA+p9SecO4HfuK3ZXdKMbRpjDrxCIKkN6gO19DzQIBEJFZE7RORH91i2zp1U9xzUkFdwLr5TgZ/g3ND8wp3WH7jH5ybrNpxjV2PHXn+O83Nwinu+ICJbROROEQlvYH11z4Gw9/hfe17a5TNtI/vbM87P/dVW52vfWJskIpNE5BMRyReRnThFMFt6LdGqawf2/x6jmvH732fbboJZSPPO2XWvs34hIkt8fouj8P83Dk49fd8bNLW/zWSc/eK7Pd/z98c41W4eAPJEZLaIxDVju52KJXimzanqalU9F6eowj+BV0SkC/s/fQPYgnPiqtUP5/F9Hk7Z7D61E0SktjjfPpur8/4hnLt0g1U1DvgrzkmwLTQWa6NE5HCccuxn4xRFSMApQ96S2LYAiSISWyeWze7rUpziDbWa05plfd+RrwL2PvWqb9st2cY1OE8yJ7nf2U/c8dLA/M1R9zckvu9VNVdVL1PVVJw75A+KSHortmeMaV9f4RRZa6xJ99YcAxtdl3vjKbmBec/DaejlaJwbeGm1i7l/Gz2WqeoO4AOc88R5OE+EapfZCPzK90arqkar6pfN/0j7bLNSVf+uqiNw6qydhFMEsr54654DYe/xPwfnvOS73/vWt0mf103tr9ZoLFZ/PIfzBLKvqsbjFMdsaVyNXTs05/dVn6bOj/ts270W604Lz9nuE+NHgStwirsm4JSEaovzdT7OfunjM26f35Cq3quqB+NUsRmCU5fW1MMSPNPmROTnIpKsqjU4lY3BqXSbj1M8z7f/sueBP4jIABHpivPE7UX37s4rOEU/JotIBPB3mj7AxuKU+y8RkWHAb9rsgzUea1NicQ5c+UCYiNyIUw+w2VR1I/AlcLuIRInIGJynpv91Z1kCnCAiiSLSE7iqGavPA7o3VKTIfaL4EvAPEYl1D/ZX41TSb6lYnLudO0QkEbipnpha2ufd/4DRInKqe8fzcnwu9kTkLBGpPZlsxzk5Ve+/GmNMR6BOUfobgQfc/+sYEQkXkeNF5E53ttYcA+tahfPE5ET3ydYNOMXr6xOLU9etEOei/bY60/05lj2Hk2Cdwd7imeAkGNeJyEgAEYkXkbOa80HqIyJHishoN7EowrmBV3sMrBvvO8AQcbqoCBORn+HUk3pbVdfjNK42S0QiROQwmi4O2dT+ao0GY/Vz+VicJ5JlIjIRJxltqcauHZrz+6pPU7+p54CLRWSciES62/7arV7RErU36/MBRORinCd4vvH0ca/ZmsW9vngN5zcU417D1d5sQEQOcZ+shuMkxrUNuph6WIJn2sNxwHIRKcFpOfIct5z6Lpz6VfPdR/uHAk/gFBH5HKfydBlwJYA6deSuxKl/loPTYtNWnBNCQ/6IcyAuxrnL9GIbfq4GY/XD+zj1+1bhFJcoo/7iK/46F+du5xacegk3qeqH7rQ5OPUt1uHcDfZ7H6jqSpyT0Rr3O0qtZ7YrcQ6ua4AMnBPIEy36FI7/w2kIoACn4YP36ky/BzhTRLaLyL3NWbGqFuBUCL8T5yJiBM5FSO1v6BDga/e3Ohf4vaqubekHMca0P1W9G+fG0g04F5obcZ4o1Pbj1eJjYD3b2olTJeExnKcepexbzNvXMzjH9804DXbVrX/+ODDCPbY21OfYXGAwkOfWnauN43WcEjEvuMUZfwCOb9GH2ldPnJupRTgNbn3G3ht2+xx7VbUQ5wnfNTjH0z8BJ7nHWXDqex3mTrsVZ783dr5uan+1mB+xNuW3wM0iUoxzQ+GlVoTT2HVOc35f9bkduMH9Tf2x7kRV/QinBfJXca6jBuHU7WwRt17cXThP0vNwGqKZ7zPLxzgtfeaKiL/72tcVOE9za1vZfp69v6E4nOu67extXf3fLdhGpyB7n/4b07G5d7524BS/tItw02wiEoJz8jxfVT/xOh5jjAlWIvIisFJV65bKMMYvIvJPoKeqXuh1LIHGnuCZDk1EZriP6rvg3Kn5nr0VsY1pkogcK07ro5HsrZPZZneKjTHG7ClCN0hEQkTkOJz6dQ09qTRmPyIyTETGiGMiTvWT172OKxBZr/SmozsF5zG94BStO0ftsbNpnsNwipFG4BQDOrW2uWdjjDFtpidOHaruOCUlfqOqi70NyQSYWJximak4VXLuwul31zSTFdE0xhhjjDHGmCBhRTSNMcYYY4wxJki0W4LnNt/+jYgsFacD4b+74weIyNcislpEXmxJU6rGGGOMMcYYY/bXbkU03Q6Fu6hqidtnRQbwe5ymjV9T1RdE5GFgqao+1Ni6kpKSNC0trV3iNMYY03EsWrSoQFWb09Fvp2bnR2OM6Tz8PUe2WyMrbkMYJe7bcHdQ4Cj2dhj5NDALaDTBS0tLY+HChe0TqDHGmA5DRNZ7HUMgsfOjMcZ0Hv6eI9u1Dp6IhIrIEpyWcD4EfgR2qGqVO8smoHcDy84UkYUisjA/P789wzTGGGOMMcaYoNCuCZ6qVqvqOKAPMBEYXt9sDSw7W1UnqOqE5GQrrWOMMSawicgTIrJVRH5oYLqIyL0iki0iy0TkoAMdozHGmMB3QFrRVNUdwKfAoUCCiNQWDe0DbDkQMRhjjDEeewo4rpHpxwOD3WEmTVRfMMYYY+rTnq1oJotIgvs6GjgayAQ+Ac50Z7sQ68DQGGNMJ6CqnwPbGpnlFOAZdSzAuSHa68BEZ4wxJli0WyMrQC/gaREJxUkkX1LVt0VkBfCCiNwKLAYeb8cYOrV5K/KoUeWnI3t6HYoxxpim9QY2+ryvraee4zuTiMzEecJHv379DlhwxpjOo6q6huKyKorKKp2/uyspqvO+uKyK0vIqatqpRf5gc9pBvZk8KOmAbKs9W9FcBoyvZ/wanPp4ph2pKjfNXU5ZZTVHDetBWKj1aW+MMR2c1DNuvysnVZ0NzAaYMGGCXVkZ0wlU1yg5O3ezY1flniTLN9FyxlVStLuK4vK9SVhVTfMOEZVuYrerorrJebtGhtElMpRQqe/QZeqanN79gG2rPZ/gGQ+tL9zF5h27AfhqTSGHD7aGaowxpoPbBPT1eW/11I3phHburmRlThErc4vJzCkiM7eYVbnF7K5sOOnqEhFKbFQ4cdFhxEaFk9glgv7duxAe2rzkKzwkhNioMOKiw52/Uc7f2nXHRYUTFxVO16gwQkMsseuoLMELUl9kFwAQERrC3CVbLMEzxpiOby5whYi8AEwCdqpqThPLGGMCVHWNsq6wlMycIlbmOMncytziPTfoARJiwhneM45zJvZlaEos3bpE7Em64t0krGtkmJXUMvuwBC9IZazOp3dCNJMGJPLe8lxuPW0UkWGhXodljDGdlog8D0wDkkRkE3ATEA6gqg8D7wAnANnALuBibyI1xrSFmholv6ScTdt3s3nHbrbs2M1m9/Xm7btZv62UssoaAEJDhEHJXTi4fzd+fmh/hvWKZXjPOFLiIhErAmmayRK8IFRdo3z5YyEnjOrFcaN78trizXy+qoBjRqR4HZoxxnRaqnpuE9MVuPwAhWOMaaWKqhpydjrJ2iaf5G3LDudvzo4yKqpr9lkmLiqM3t1i6JsYw9TBSQzvFcewnrGk9+hKVLjdiDdtwxK8ILRs0w6Ky6qYOjiJqelJdIsJZ+7SLZbgGWOMMcb4qaS8yk3adu2XxG3evpv8knLqNiDZIzaS3t2iGd07nuNG9aRPQjS9u0XTOyGG1IQoYqPCvfkwplOxBC8IZawuQASmpCcRHhrC8aN78fp3m9lVUUVMhH3lxhhjjOkcamqUrcXlFJSUN9jUf30tUG4vraCorGqfdYWHCqkJ0aTGR/OTIcn0dpO3PgnRpCZE0yshyqrDmA7BrvaD0BfZBYxMjSOxSwQAM8ak8tzXG/gocyszxqZ6HJ0xxhhjTNsor6omZ0fZnqdqe5+yOa2J5+4so7K64a4CukaG7dNaZHLXSAYldyU+Opxe8bVP36Lp0y2a5K6RhFjLkSYAWIIXZErLq1i8YTuXTB2wZ9zEAYmkxEUyd+kWS/CMMcYYE1BUldyiMqelyVynxcmN251ik1uLy/eZVwRSYqPo3S2a8X270XuMk6D1iI3cp+l/a+rfBDNL8ILMN2u3UVmtHJ6+t1uE0BDhxNGpPLtgPTt3VxIf3XblvxesKWREahxxVqbcGGOMMa1UVlnNqjy3/7ecYlbmOl0H7NhVuWee3gnRpCXFMG1o8p66bU5RyRh6xkcREWZdBpjOzRK8IPPF6gIiw0KYkNZtn/Ezxvbiiflr+WB5LmdN6NvA0s3zw+adnDN7AZcfOYhrjx3WJus0xhhjTOexJr+Ed3/IZUVOEZk5RawrKKXGLVEZHR7K0J6xHD+qF8N7xTKsZxxDe8a26Y1qY4KRJXhBZn52AYekJe7X1O64vgn0TYxm7tItbZbg3f9xNuAkldce2yarNMYYY0yQq65RPs3aytNfrefzVfkA9EuMYVjPWE4ak8oIN5nrlxhjdd6MaQFL8ILI1qIysvKKOe2g3vtNExFmjEnlkc/XUFBSTlLXyFZta1VeMe8tzyWpayTfb97Jjl0VJMREtGqdxhhjjAleO3ZV8NLCjcxZsJ6N23aTEhfJ1ccM4ZyJfekRG+V1eMYEDSukHEQysgsAmJqeVO/0GWNTqa5R3v0+p9XbeuCTbGIiQrnj9NGowpc/FrZ6ncYYY4wJPsu37OTPryxj0m0fcds7K+kVH80D5x1Exp+P4nfTB1tyZ0wbsyd4QSRjdQGJXSIY0Suu3unDesYyuEdX3lqawwWHpbV4O+sKSnlr6RYuO3wg04Ym0zUyjC9WF3DC6F4tXqcxxhhjgkdFVQ3vLc/lmS/XsXD9dqLDQzn9oD784rD+DG/gOsUY0zYswQsSqkpGdgGTB3VvsLy6iDBjbCp3f7iKnJ276RUf3aJtPfTpj4SHhnDp4QMICw3h0IHdycjOb034xhhjjAkC+cXl/Pfr9Tz39Qa2FpfTv3sMN5w4nLMO7kt8jDWOYsyBYAlekFi9tYStxeUcPrj+4pm1ahO8t5fmcNlPBjZ7O5t37ObV7zZx/qR+e4pUHD44iXmZeWwo3EW/7jEtit8YY4wxgauyuoanv1zHfz5cRWlFNUcOTeafk9M4YnCyNZRizAHWbgmeiPQFngF6AjXAbFW9R0RmAZcBtY98/qqq77RXHJ3FF6vd+neDkxudb0BSF0b3juetZVtalOA98tmPiMDMIwbtGTfVTSq/yM7n/O79m71OY4wJNCIyGUjD5zyqqs94FpAxHvp6TSE3vrmcrLxipg1N5m8njWBQclevwzKm02rPJ3hVwDWq+p2IxAKLRORDd9p/VPXf7bjtTmd+dgEDkrrQO6HpYpczxvbitndWsq6glLSkLn5vY2tRGS98u5EzDuqzz3YGJnUhNT6KjNUFnD/JEjxjTHATkTnAIGAJUO2OVpybmsZ0GluLy7j9nZW8vngzvROimX3BwRwzIgURe2JnjJfaLcFT1Rwgx31dLCKZwP7t95tWq6iqYcGaQs44qI9f8580JpXb3lnJW0u3cOX0wX5v57GMtVRV1/Brn6d34NTtm5KexAcr8qiuUUKtKIYxJrhNAEaoqnodiDFeqKqu4dkF67nrg1WUVVVz+ZGDuOLIwURHhDa9sDGm3R2QbhJEJA0YD3ztjrpCRJaJyBMi0u1AxBDMFm/Yzq6K6j1FJZuSmhDNIWndeGvZFr+3sa20gmcXrOfksan1PvWbOjiJnbsr+WHzTr/XaYwxAeoHnOoHxnQ6i9ZvY8b985n11grG9Uvg/at+wrXHDrPkzpgOpN0TPBHpCrwKXKWqRcBDOEVbxuE84burgeVmishCEVmYn28tNDYmI7uAEIHDBnX3e5kZY1NZlVdCVm6xX/M/OX8tuyqqufzI9HqnT3H73qvti88YY4JYErBCRN4Xkbm1g9dBGdOeCkrKufblpZzx0Ffs2FXBg+cfxDOXTGSg1bUzpsNp11Y0RSQcJ7n7r6q+BqCqeT7THwXerm9ZVZ0NzAaYMGGCFYNpREZ2AWP7JhAX5X/zwyeM7sWsucuZu3Qz1/Yc1ui8RWWVPPXlOo4f1ZPBKbH1zpPUNZIRveL4YnV+g0mgMcYEiVleB2DMgVJdozz3zQb+9d5KdlVU86sjBvK7owbTJdIaYjemo2q3J3ji1LB9HMhU1bt9xvv2hn0aTlEX00I7d1eydOMODk/3r3hmraSukUxJT+KtpTk0VY1kzlfrKS6rajJxmzo4iUXrt7OroqpZsRhjTCBR1c+AlUCsO2S644wJKl/+WMApD2Twtzd+YGRqPO9ddTjXHT/ckjtjOrj2LKI5BbgAOEpElrjDCcCdIvK9iCwDjgT+0I4xBL2vfiykRpvuHqE+M8amsmHbLpZtarje3K6KKh77Yg1HDk1mVO/4Rtc3NT2Jymrlm7Xbmh2LMcYEChE5G/gGOAs4G/haRM70Nipj2s7K3CIuevIbznv0awpLKrjnnHE8d9kk0nvUX4rHGNOxtGcrmhlAfc0pWp93bWh+dgExEaGM65vQ7GWPHdmT61//nrlLtzC2geWf+3oD23dVcsVRTbe2OXFAIhFhIWSsLmDa0B7NjscYYwLE9cAhqroVQESSgXnAK55GZUwrbd6xm7s/WMVrizcRGxnGX44fxkWT04gKtwZUjAkk9ow9wGVkF3DowO5EhDX/YWx8dDhHDOnB28u2cP0Jwwmp071BWWU1sz9fw+RB3Tm4f9ONnUaFh3JIWjdraMUYE+xCapM7VyF+lIgRkeOAe4BQ4DFVvaPO9H7A00CCO89fVNVuipp2t3NXJQ9+ms2TX64D4LLDB/LbaYNIiInwNjBjTItYghfANm3fxdqCUi44tOWdi588LpV5mXl8s24bhw7ctxXOlxdtYmtxOf/3s3F+r29KehJ3vpfF1uIyesRGtTguY4zpwN4TkfeB5933P6OJ0ikiEgo8ABwDbAK+FZG5qrrCZ7YbgJdU9SERGeGuM62tgzemVlllNc98tY4HPvmRorJKThvfm6uPGUKfbjFeh2aMaQVL8AJYxmrnSdnhfvZ/V5+jh/cgOjyUt5Zu2SfBq6yu4eFPf+SgfgnN6n7h8PRk7iSLL7MLOXW89WtvjAk+qnqtiJyBU9dcgNmq+noTi00EslV1DYCIvACcAvgmeArEua/jAf87KzWmGaprlDcWb+buD1execdujhiSzJ+PG8aI1LimFzbGdHiW4AWwjOwCUuIiSe/R8j5oYiLCmD68B+/+kMusk0cSHuqUMnp98WY279jNraeOwmkQ1T8jU+PoFhPOF6sLLMEzxgQtVX0Vpxsgf/UGNvq83wRMqjPPLOADEbkS6AIcXd+KRGQmMBOgX79+zQjBdHaqymer8rnj3ZWszC1mdO94/nXmGCY3syVuY0zH1u4dnZv2UVOjfPljIVPSk5qVgNXn5LGpbCutYL5bd666Rnno0x8ZmRrHtKHNa50zJESYnJ5ERnZ+k90vGGNMIBGRDPdvsYgU+QzFIlLU1OL1jKt7kDwXeEpV+wAnAHNEZL/ztKrOVtUJqjohObn5LSibzklVueXtTC568lt2VVRz37njefPyKZbcGROELMELUCtyithWWtGq4pm1jhiaTGxUGG8tzQHgf9/nsLaglCuOTG9R8jg1PYm8onJ+zC9pdWzGGNNRqOpU92+sqsb5DLGq2lTZtk1AX5/3fdi/COalwEvuNr4CogC7+jatpqrc/u5Knpi/losmpzHv6iOYMTZ1v8bVjDHBwRK8AFVjp/S6AAAgAElEQVTbUuWUQa0/90eGhXLcyJ58sDyXsspqHvg4m/QeXTl2ZM8WrW+qezfwi9XWmqYxJviIyBx/xtXxLTBYRAaISARwDjC3zjwbgOnu+objJHj5rY/YdHZ3fbCK2Z+v4ReH9eemGSNa1PK2MSZw2H94gMpYXcDQlFh6xLVNS5UzxqZSXF7F9a//QFZeMVccmd7iO3t9E2NI6x6zpxEYY4wJMiN934hIGHBwYwuoahVwBfA+kInTWuZyEblZRE52Z7sGuExEluK00HmRWll300r3frSa+z/J5tyJfZk1Y2Srq3UYYzq+ZjWyIiLdgL6quqyd4jF+KKus5pt121rVPUJdkwd1p3uXCF79bhP9EmM4aUyvVq1vSnoSbyzeTGV1zZ6GW4wxJpCJyHXAX4Fonzp3AlQAs5ta3u3T7p064270eb0Cp2VOY9rEg59mc/eHqzjz4D7849TRViTTmE7Cn45ZPxWROBFJBJYCT4rI3e0fmmnIt+u2UVFVw9Q2qH9XKyw0hBNGO0ndb6cNIqyVSdnhg5MorahmycYdbRGeMcZ4TlVvV9VY4F916t91V9XrvI7PGF+PfbGGO9/L4pRxqfzzjDGW3BnTifjzBC9eVYtE5JfAk6p6k4jYEzwPZWQXEB4qTBqQ2KbrvezwgUSFh3D6QX1ava7DBiURIk49vEPS2jZOY4zx2Lsi8pO6I1X1cy+CMaauZ75ax63/y+SE0T2566yxhFpyZ0yn4k+CFyYivYCzgevbOR7jh4zVBRzUrxsxEW3bjWG/7jFcf+KINllXfHQ4Y/okkLE6n6uPGdIm6zTGmA7iWp/XUTidmC8CjvImHGP2eu7rDdz45nKOGZHCPeeMb3WJHGNM4PHnv/5mnErh2ar6rYgMBFa3b1imIYUl5SzfUtQm3SO0t6npSSzdtJOiskqvQzHGmDajqjN8hmOAUUCe13EZ8/LCjVz/xvccOTSZ+88bb3XgjemkmvzPV9WXVXWMqv7Wfb9GVc9o/9BMfeb/WAg4jZh0dFMHJ1FdoyxwYzbGmCC1CSfJM8YzbyzezJ9eXcbU9CQe+vnBRIaFeh2SMcYjTZbxE5EBwJVAmu/8qnpyQ8uY9jN/dQFxUWGM6ZPgdShNcoqRhpKRXcBPW9innjHGdDQich9Q231BCDAOpxEyYzzxv2U5XP3SEg4d0J3ZF0wgKtySO2M6M38qcb0BPA68BdS0bzimMapKRnYBkwclBUSF6YiwECYNSLT+8IwxwWahz+sq4HlVne9VMKZze395Lr9/YTEH9+/G4xdNIDrCkjtjOjt/ErwyVb233SMxTVpbUMrmHbv5zbRBXofitynpSXySlcnmHbvpnRDtdTjGGNNqqvq0iEQAw3Ce5GV5HJLppD5ZuZUrnvuO0X3iefLiiW3e+JoxJjD5U/v2HhG5SUQOE5GDaoemFhKRviLyiYhkishyEfm9Oz5RRD4UkdXu326t/hSdxPxs50nY1ACof1fr8MHJgFO01BhjgoGInAD8CNwL3A9ki8jx3kZlOpuv1xTy62cXMaxnHE9dPJGukZbcGWMc/hwNRgMX4DT/XFtEU2m6Oegq4BpV/U5EYoFFIvIhcBHwkareISJ/Af4C/LklwXc2X6wuoE+3aPp3j/E6FL8NSelKj9hIvsgu4OxD+nodjjHGtIW7gSNVNRtARAYB/wPe9TQq02ks37KTXz69kD7donn6konER4d7HZIxpgPxJ8E7DRioqhXNWbGq5gA57utiEckEegOnANPc2Z4GPsUSvCaVlFfx1ZpCThrTC5GOX/+ulogwNT2JT1flU1OjhARA3UFjjGnC1trkzrUG2OpVMKZzWVtQyoVPfENsVBhzLp1EYpcIr0MyxnQw/hTRXAq0qslGEUkDxgNfAylu8lebBPZoYJmZIrJQRBbm5+e3ZvMBb0PhLk5/cD67Kqo5eWxvr8NptinpSWwrrSAzt8jrUIwxpsVE5HQROR1YLiLviMhFInIhTiNk33ocnukE8orKuODxr6lReObSSaRa3XZjTD38eYKXAqwUkW+B8tqR/naTICJdgVeBq1S1yN+nT6o6G5gNMGHCBG1i9qC1YE0hv3l2kXMwv2Qihw3q7nVIzTbV7ZQ9Y3UBI1PjPY7GGGNabIbP6zzgCPd1PmD1yU272rmrkl88/g3bSyt4fuahpPfo6nVIxpgOyp8E76aWrlxEwnGSu/+q6mvu6DwR6aWqOSLSCyvW0qDnv9nA3974gf7dY3j8wkNIS+ridUgtkhIXxZCUrmRkF/CrIwKnBVBjjPGlqhd7HYPpnHZVVHHJ09+ytqCUpy4+JCD6wjXGeKfJBE9VPxOR/sBgVZ0nIjFAk52siPOo7nEgU1Xv9pk0F7gQuMP9+2aLIg9iVdU13Pq/TJ76ch1HDEnmvvPGExcV2BWop6Yn89+v11NWWW0dsBpjApKI/ElV76zT0fkeqvo7D8IyQa6iqobfPPsdizds58HzD2JyALWkbYzxRpMJnohcBswEEoFBOA2lPAxMb2LRKTitb34vIkvccX/FSexeEpFLgQ3AWS0LPTjt3FXJFc9/xxerC/jl1AFcd8LwgOjUvClTB3fniflrWbR+O1Ps5GSMCUyZ7t+Fjc5lTBupqVH++PJSPluVzx2nj+a4Ub28DskYEwD8KaJ5OTARp4EUVHW1iNTbMIovVc0AGspMmkoOO6U1+SX88umFbNy+izvPGBNU3QpMGtCd8FDhi9UFluAZYwKSqr4lIqHAKFW91ut4THBTVf7+1nLmLt3Cn48bxjkT+3kdkjEmQPiT4JWrakVt4ygiEkY9RVNM63y+Kp8rnvuO8NAQnrvsUA5JS/Q6pDbVJTKM8f26kZGdDwzzOhxjjGkRVa0WkYO9jsMEv3s+Ws3TX61n5k8G8usjBnodjjEmgPiT4H0mIn8FokXkGOC3OE1Cmzagqjz15TpueXsFQ1JiefQXE+ibGDgdmTfH4elJ3D1vFdtKK6zfHmNMIFssInOBl4HS2pE+jYkZ0ypPf7mO/5u3mjMP7sN1xw8LqP5vjTHe86cfvL/gNAH9PfAr4B1Vvb5do+okKqpq+Ovr3/P3t1YwfXgKr/5mctAmdwBTBiehCl/+WOB1KMYY0xqJQCFwFE7XCTOAkzyNyASNN5ds5qa5yzl6eAp3nD7akjtjTLP58wTvSlW9B3i0doSI/N4dZ1po565KLpuzkG/WbuPyIwdxzTFDCQmCxlQaM6Z3PLFRYcxbkceJo3vZScsYE6geU9X5viNEZEpTC4nIccA9OC1RP6aqd9Qzz9nALJyqEEtV9bw2idgEhE+ytnLNS0uZNCCR+88bT1ioP/fhjTFmX/4cOS6sZ9xFbRxHp/Ps1+v5Zu027jlnHNceOyzokzuAsNAQjh3ZkzeWbGH6XZ/xeMZadu6q9DosY4xprvv8HLeH2zjLA8DxwAjgXBEZUWeewcB1wBRVHQlc1TbhmkCwJr+E3z77HUN7xvLohROsSyFjTIs1+ARPRM4FzgMGuHUNasXiFE0xrZCZU0S/xBhOGdfb61AOqH+cNoqp6UnMWbCeW95ewb/eX8kpY3tzwWH9GdU73uvwjDGmQSJyGDAZSBaRq30mxdF0/7ATgWxVXeOu6wXgFGCFzzyXAQ+o6nYAVd3aVrGbjk1V+dubPxAWKjxx0SEB3/etMcZbjRXR/BLIAZKAu3zGFwPL2jOoziArt5ghKbFeh3HARYaFcur43pw6vjfLt+zk2QUbeGPxZl5cuJFxfRO44ND+nDiml925NMZ0RBFAV5xzp+8BvAg4s4llewMbfd5vAibVmWcIgIjMx0kYZ6nqe60J2ASGuUu3MD+7kJtPGUlKXJTX4RhjAlyDCZ6qrgfWA4cduHA6h/KqatYWlPLTkSleh+Kpkanx3H76aK47YRivLdrEnAXrueblpdzyvxWcPaEv50/qR//uXbwO0xhjAFDVz3Baln7KPUciIiFAV1UtamLx+srh1+1yKAwYDEwD+gBfiMgoVd2xz4pEZgIzAfr1s77RAt3O3ZXc8nYmY/rEc/6k/l6HY4wJAg3WwRORDPdvsYgU+QzFItLUicw0Ym1BKVU12imf4NUnLiqci6YMYN7VR/DcZZOYPKg7j2es5Yh/fcqFT3zDZ6vyvQ7RGGN83S4icSLSBaeIZZaINNXx+Sagr8/7PsCWeuZ5U1UrVXUtkIWT8O1DVWer6gRVnZCcnNzyT2E6hLs+yGJbaTn/OHU0oZ2gPr4xpv01mOCp6lT3b6yqxvkMsaoad+BCDD5ZucUADO1pCZ4vEWHyoCQePP9g5v/5KK46ejArc4u48Ilv+GHzTq/DM8aYWiPcJ3anAu8A/YALmljmW2CwiAwQkQjgHGBunXneAI4EEJEknCKba9oycNOxLNu0gzkL1nPBof0Z3cfqoRtj2oa1v+uBVXnFhIUIA5O6eh1Kh9UzPoqrjh7Cu7//CSECHyzP9TokY4ypFS4i4TgJ3puqWsn+xS33oapVwBXA+0Am8JKqLheRm0XkZHe294FCEVkBfAJcq6rWqFmQqq5Rrn/9B5K6RnLNsUO9DscYE0T86QfPtLGs3BIGJnchIszy66Ykdong4P7dmJe5lat/aidAY0yH8AiwDlgKfC4i/XEaWmmUqr6D88TPd9yNPq8VuNodTJD779fr+X7zTu45Z5y1mmmMaVON1cGLPJCBdCZZeUVW/64Zjh6ewoqcIrbs2O11KMYYg6req6q9VfUEdazHLVppjD+2Fpfxr/eymJqexMljU70OxxgTZBp7hPQVgIjMOUCxdAql5VVs3LaboZbg+W36cKe10Y8y8zyOxBjTmYnIz92/V9cdgN95HJ4JILe+nUl5VQ03nzISEWtYxRjTthorohkhIhcCk0Xk9LoTVfW19gsreK3eWgLAEGtgxW+DkruQ1j2GeZlbueCwNK/DMcZ0XrX9ttgB3LRYxuoC5i7dwu+mD2ZgstXFN8a0vcYSvF8D5wMJwIw60xSwBK8FVtW2oGlP8PwmIhw9PIVnvlpPaXkVXSKt6qgx5sBT1Ufcv3/3OhYTmMqrqrnxzR/o3z2G304b5HU4xpgg1VhH5xlAhogsVNXHm7tiEXkCOAnYqqqj3HGzgMuA2o7N/upWOu80svKKiQoPoV9ijNehBJTpw1N4LGMtX6zO57hRvbwOxxjTCYnIvY1NV1Urpmka9chna1hTUMrTl0wkKjzU63CMMUHKn2Yc54jI70TkFXe40m0euilPAcfVM/4/qjrOHTpVcgdOFwlDUmIJsc5Mm2VCWjfiosKYl7nV61CMMZ3XIneIAg4CVrvDOKDaw7hMAFhfWMr9n2Rz4pheHDHEOqg3xrQff8q6PQiEu3/B6cz1IeCXjS2kqp+LSFprggtGK3OL7cDeAuGhIRw5rAefrNxKdY0SagmyMeYAU9WnAUTkIuBIt/87RORh4AMPQzMdnKpy45vLiQgN4caTRngdjjEmyPnzBO8QVb1QVT92h4uBQ1qxzStEZJmIPCEi3VqxnoCzrbSC/OJyq3/XQtOHp1BYWsGSjTu8DsUY07mlsm9DK13dccbU653vc/lsVT5XHzOElLgor8MxxgQ5fxK8ahHZUxNYRAbS8qIoDwGDcIqz5AB3NTSjiMwUkYUisjA/P7+h2QLKqjyngRVrQbNljhiSTFiIMM+6SzDGeOsOYLGIPCUiTwHfAbd5G5LpqIrLKrn57eWM6BXHLw7r73U4xphOwJ8E71rgExH5VEQ+Az4GrmnJxlQ1T1WrVbUGeBSY2Mi8s1V1gqpOSE4OjiKNtQneMEvwWiQ+OpyJAxKtPzxjjKdU9UlgEvC6OxxWW3zTmLr+8+FqthaX84/TRhEW6s9llzHGtE6TdfBU9SMRGQwMBQRYqarlLdmYiPRS1Rz37WnADy1ZT6DKyi0mPjqcHrGRXocSsKYPT+GWt1ewoXAX/bpbS6TGGG+oai7wptdxmI5t+ZadPPXlWs6b2I/x/TpVrRRjjIf8upWkquWqukxVl/qb3InI88BXwFAR2SQilwJ3isj3IrIMOBL4Q4sjD0BZucUMTYlFxBoIaamjh/cAsGKaxhhjOrSaGuX6138gsUsEfzp2mNfhGGM6kXbrMVpVz61ndLP70wsWqkpWXjGnjLN6+K3Rv3sXBvfoykcr87hk6gCvwzHGGGPqNWfBepZs3MHdZ48lPsaf3qWMMaZtWGHwAyS3qIzisiprQbMNTB+ewtdrtlFUVul1KMaYTkpEporIxe7rZBGxO05mjx827+Qf/8tk2tBkThvf2+twjDGdTJMJnoh85M8407isXLcFTUvwWu3o4T2oqlE+ywqO1lWNMYFFRG4C/gxc544KB571LiLTkRSVVXL5c9/RvWsEd589zqplGGMOuAYTPBGJEpFEIElEuolIojukYf39NFttC5pDrQXNVhvfrxuJXSKsNU1jjFdOA04GSgFUdQv79otnOilV5S+vLmPT9t3cd+54ErtEeB2SMaYTaqwO3q+Aq3CSuUU4LWgCFAEPtHNcQScrt4SUuEgSYuxg31qhIcKRQ3swLzOPquoaa3baGHOgVaiqiogCiEgXrwMyHcMzX63nne9zue74YUxIS/Q6HGNMJ9XglbGq3qOqA4A/qupAVR3gDmNV9f4DGGNQyMorsuKZbejo4T3YubuSheu3ex2KMabzeUlEHgESROQyYB5O366mE1u2aQe3/m8FRw3rwWWHD/Q6HGNMJ+ZPP3j3ichkIM13flV9ph3jCirVNcrqvBIuOLS/16EEjcOHJBMRGsJHmXkcOrC71+EYYzoRVf23iByDU6JlKHCjqn7ocVjGQzt3O/XukrtGctdZYwkJsXp3xhjvNJngicgcYBCwBKh2RytgCZ6fNmzbRXlVDUOs/l2b6RoZxqGDuvNR5lauP3GE1+EYYzoREfkD8LIldQacend/emUpOTvKeOnXh9HN6t0ZYzzmTz94E4ARqqrtHUywqm1Bc5gleG3q6OE9uPHN5fyYX8Kg5K5eh2OM6TzigPdFZBvwAvCKqlqrT53UE/PX8f7yPG44cTgH9evmdTjGGONXP3g/AD3bO5BgtiqvGBFI72FJSFs6algPAGtN0xhzQKnq31V1JHA5TkNkn4nIvKaWE5HjRCRLRLJF5C+NzHemiKiITGjDsE07WLxhO7e/k8kxI1K4dKp1hWiM6Rj8SfCSgBUi8r6IzK0d2juwYJKVW0y/xBhiIvx5YGr81adbDMN7xTEvc6vXoRhjOqetQC5QCPRobEYRCcVpgfp4YARwrojsV75cRGKB3wFft3m0pk3t2FXBFc8tpmd8FP8+c6z1d2eM6TD8yThmtXcQwS4rr9ha0GwnRw/vwQOfZLO9tMLqPRhjDggR+Q3wMyAZeAW4TFVXNLHYRCBbVde463gBOAWou9wtwJ3AH9s0aNOmVJU/vryUrcVlvPLrycTHhHsdkjHG7NHkEzxV/ay+4UAEFwzKq6pZW1DKUEvw2sX04SnUKHy6yp7iGWMOmP7AVao6UlVv8iO5A+gNbPR5v8kdt4eIjAf6qurbbReqaQ+PfrGGeZlb+esJwxnbN8HrcIwxZh9NJngiUiwiRe5QJiLVIlJ0IIILBmvyS6muUWtBs52M6R1PcmykFdM0xrQ7EYlzX94JbBCRRN+hqcXrGben8TIRCQH+A1zjRxwzRWShiCzMz8/3N3zTRhat38Y/38vi+FE9uWhymtfhGGPMfvzpB2+fzERETsUpamL8sCrPWtBsTyEhwvRhPXh7WQ4VVTVEhPlTrdQYY1rkOeAkYBFOcuabtCnQWO/Wm4C+Pu/7AFt83scCo4BP3bpcPYG5InKyqi70XZGqzgZmA0yYMMFauD6AtpU69e56J0TzzzPHWL07Y0yH1OyrYVV9AziqHWIJSlm5xYSHCmndu3gdStCaPjyFkvIqvlm7zetQjDFBTFVPcv8OUNWB7t/aobHkDuBbYLCIDBCRCOAcYE+DZaq6U1WTVDVNVdOABcB+yZ3xTk2NcvVLSygsqeDB8w8iLsrq3RljOiZ/Ojo/3edtCE6/eHbH0E9ZucUMTOpqT5ba0dT0JCLDQpiXmcfUwUleh2OMCXIi8pGqTm9qnC9VrRKRK4D3gVDgCVVdLiI3AwtV1Vqn7uAe+XwNn2blc8spIxnVO97rcIwxpkH+tKI5w+d1FbAOp+Uv44esvGLGW8en7So6IpSp6UnMy8zjphkjrMiMMaZdiEgUEAMkiUg39hbRjMPpD69RqvoO8E6dcTc2MO+0VgVr2tTGbbv4z4erOH5UT35+aH+vwzHGmEb5Uwfv4pasWESewKmrsFVVR7njEoEXgTScRPFsVd3ekvUHgpLyKjZt3805h/RtembTKtOHp/DRyq2syithqNV3NMa0j18BV+Ekc4vYm+AV4fRxZ4LUbe9kEhoizDp5pN1ENMZ0eP60otlHRF4Xka0ikicir4pIHz/W/RRwXJ1xfwE+UtXBwEfu+6C12m1gZWjPuCbmNK01fbjTx/C8zDyPIzHGBCtVvUdVBwB/rFMHb6yq3u91fKZ9LFhTyLs/5PLbaYNIiYvyOhxjjGmSPxXDnsSpCJ6K02fPW+64Rqnq50DdVi9OAZ52Xz8NnOp3pAGotgVN6wOv/aXERTGmT3yzE7zqGuWTlVv5ZKV1s2CM8Y+q3icio0TkbBH5Re3gdVym7VXXKDe/tYLeCdFc9pOm2tExxpiOwZ86eMmq6pvQPSUiV7VweymqmgOgqjki0qOhGUVkJjAToF+/fi3cnLdW5hYTHR5Kn27RXofSKUwflsL/fbSKgpJykrpGNjpvYUk5Ly7cyH8XbGDzjt2IwJMXHcK0oQ3+JI0xBgARuQmYBozAqVN3PJABPONhWKYdvLJoIytyirjv3PFEhYd6HY4xxvjFnyd4BSLycxEJdYefA4XtHZiqzlbVCao6ITk5ub031y5W5RUzJKUrISFWXv9AOHpED1Th4waexqkqi9Zv46oXFnPY7R9z53tZ9EuM4d5zxzOsZxy/e34xawtKD3DUxpgAdCYwHch166mPBRq/q2QCTnFZJf96fxUH9+/GSWN6eR2OMcb4zZ8neJcA9wP/weke4Ut3XEvkiUgv9+ldLyCoy8Vl5ZZw5NDATE4D0YhecaTGR/FRZh5nT9jbsE1peRVvLtnCnAXrycwpIjYyjPMm9eP8Sf0Y7BafHd83gZPvz2DmMwt5/fIpdI3051/DGNNJ7VbVGhGpEpE4nHOZld8LMg988iMFJeU8fuEEa1jFGBNQ/GlFcwNwchttby5wIXCH+/fNNlpvh1NYUk5BSbm16HgAiQhHDe/Bq4s2U1ZZzabtu3l2wXpeXbSJ4vIqhvWM5bbTRnPKuFS61Eng+ibG8MB5B3HBE99w9YtLePjnB9uTV2NMQxaKSALwKE5rmiXAN96GZNrShsJdPJGxljMO6sPYvgleh2OMMc3iT0fnA4Arcbo22DO/qjaa9InI8zh1FJJEZBNwE05i95KIXApsAM5qaeAd3aq8EgBL8A6wo4en8OyCDZz6wHxW5hYTERrCCaN7csFh/TmoX7dG78JOTk/i+hOGc/PbK7jv42x+f/TgAxi5MSZQqOpv3ZcPi8h7QJyqLvMyJtO2bn/X6RbhT8cN9ToUY4xpNn/Kob0BPI7TemaNvytW1XMbmDTd33UEMmtB0xuHDuxOUtdIisuq+NNxQzl7Qt8mG1zxdfGUNH7YspP/zFvFiNQ4jhmR0o7RGmMCiYgc1Ng0Vf3uQMZj2kdttwjXHDPEukUwxgQkfxK8MlW9t90jCTIrc4tJiAknOdbq3R9IUeGhZPz5SMJDQwhtQRFLEeG200aTvbWEP7y4hDcun0x6D0vSjTEA3NXINAWOOlCBmPZh3SIYY4KBPwnePW6T0B8A5bUj7U5l45wWNGOtYrYHWtuUdVR4KA///GBOvj+Dy55ZxBuXTyE+OryNojPGBCpVPdLrGEz7enmhdYtgjAl8/iR4o4ELcO5M1hbRtDuVjVBVVuUWc+r43l6HYlooNSGaB88/mPMeXcBVLyzmsQsPadETQWNM8GmoU3NVtX7wAlhxWSX//iCLCdYtgjEmwPmT4J0GDFTVivYOJljk7CyjuLzKGlgJcBMHJDLr5JHc8MYP/OfDVfzxWKtsb4wB4BCf11E4dcu/wzo6D2hOtwgVPHHRIVb6xhgT0PxJ8JYCCQR5n3VtKau2gRVL8ALe+ZP6sXzLTu7/JJsRqXGcMNru6hrT2anqlb7vRSQemONROKYNrC8s3dMtwpg+1i2CMSaw+ZPgpQArReRb9q2D11Z94wWdVblOgjfEGucIeCLCrJNHkpVbzDUvLWVAUheG94rzOixjTMeyC7B+VQLY7e+sJCzUukUwxgQHfxK8m9o9iiCTlVtMz7go4mOsYY5gEBnmNLpy0n0ZzJyzkLeumEpCTITXYRljPCIib+HURQcIAUYAL3kXkWmNr34s5L3lufzxp9YtgjEmODSZ4KnqZ77vRWQKcB7wWf1LmKy8YoZY8cyg0iMuikcuOJifPbKAK59fzJMXHUJYaIjXYRljvPFvn9dVwHpV3eRVMKblqmuUm992ukX45eHWLYIxJjj4dYUqIuNE5E4RWQfcCmS2a1QBrLpGWb21hKEpXb0OxbSx8f26cetpo/hidQF3vp/ldTjGGI+o6mfuzc/FOOfDXSKS6HFYpgVeXriRzJwirjthmHWLYIwJGg0+wRORIcA5wLlAIfAiINYPUOPWF5ZSUVXD0J5WTysYnT2hL8s372T252s4bFB3jhzaw+uQjDEHmIjMBG4BduN0HyQ4RTbtEVAA8e0W4URrQMsYE0Qae4K3Eqfp5xmqOlVV7wOqD0xYgWtVbQuaKVZEM1hdf+IIBiZ3Ydbc5ZRV2r+EMZ3QtcBIVU1T1YGqOkBVLbkLMPd/kk1BSQU3zhhh3SIYY4JKYwneGUAu8ImIPCoi03HuUppGrEW4xIAAABsWSURBVMwtRgTSe1gRzWAVERbCLaeMYn3hLh75bI3X4RhjDrwfcVrONAFqZW4RT2as48yDrVsEY0zwaTDBU9XXVfVnwDDgU+APQIqIPCQiPz1A8QWcVXnF9E+MITrCyvIHsynpSZw0phcPfprNhkK7zjOmk7kO+FJEHhGRe2uHphYSkeNEJEtEskXkL/VMv1pEVojIMhH5SET6t0v0nVxhSTmXPrWQbl3CrVsEY0xQarKRFVUtVdX/qupJQB9gCbDfick4snKLGWLFMzuFG04cQViIMOut5ahq0wsY8//t3Xl8VfW57/HPk4SAkMg8EwIig4ogEqZii/VSh94qYkWlBe2rdZ5qz2mr7elgx2sHe3rt7bl1amsZbJUq4lBtrbQqEiBhHgUhgRAgQBjCEEKyn/PH3sE0h4TEDCtZ6/t+vfYre6+91m8/P1bYT561fuu3JCweB94CsoHcKo8amVky8GvgKuK3VZhuZudXW20FkOXuI4B5wE8bOe7IKyuPcefsXPYdOcGTN2fRI123RRCR8KnXPO/uXuzuj7v7ZU0VUGtWerKCvP3HGKZbJERCr47teGDyEN7aWMSbG4qCDkdEmk+5u/+bu//O3Z+pfJxhm7HAFnff6u5lwB+BKVVXcPeF7l45JCCb+EFVaSTuzrfmr2FZ3gF+Pm2khmaKSGjpRl6NaOveo1TEXPfAi5AvTBzAkJ5pPLxgHcfLNOGKSEQsNLPbzay3mXWpfJxhm77AjiqvCxLLavIl4C8NDVQ+9PS723gup4D7LzuXq0f2CTocEZEmE0iBZ2Z5ZrbGzFaaWU4QMTQFzaAZPW2S4xOu7Dx4nF8v3BJ0OCLSPD5H4jo8PhyeeaZcdrpJyk47ttvMZgBZwM9qeP92M8sxs5y9e/fWOegoW7ipiB+/toGrhvfigclDgg5HRKRJ1XgfvGbwSXffF+DnN7qNu0tok2wM6NYh6FCkGY07pytTR/Xlibe3ct3FfTmnu2ZQFQkzdx/4ETYrADKqvO4HFFZfycwmA/8BTHL3EzV8/hPAEwBZWVm6APgMthSVcP/cFQzrdTaP3jCSpCRNCC4i4aYhmo3o/T0lDOqeRptk/bNGzTc+PYy2KUl8d4EmXBEJOzO7+XSPM2y2DBhsZgPNLBW4CVhQrd1RxCdwucbddWFvIzhwtIwvPZND2zbJPHlLFu1TgzyuLSLSPIKqRBz4q5nlmtntAcXQ6DSDZnT1SG/Hv18+hHc27+Mva3cHHY6INK0xVR4fBx4GrqltA3cvB+4F3gA2AM+5+zoz+76ZVW77MyANeD5xCcOCGpqTOjhZEePuOcvZdbCUx2eOpm+ns4IOSUSkWQR1KGuiuxeaWQ/gb2a20d3frrpCovC7HaB///5BxFgvJaUn2XnwOJ8b1/JjlaYxY3wmz+UU8INX1jNpSHc6tNWRYpEwcvf7qr42s47ArDps9xrwWrVl36nyfHJjxSjwvZfXsXjrfh6dNpLRmZ2DDkdEpNkEcgbP3QsTP4uAF4lPH119nSfcPcvds7p3797cIdbb5qIjgCZYibKU5CR+cO1wdh0q5bG3Ngcdjog0n2PA4KCDkA/NWpzH7Ozt3DHpHD47WnebEJFoafYCz8w6mFl65XPgcmBtc8fR2DbtTsygqVskRNrozM7ckNWPp9/ZxpaikqDDEZEmYGYvm9mCxOMVYBPwUtBxSdy7m/fx8MvrmXxeD75+xbCgwxERaXZBjCHrCbxoZpWfP9fdXw8gjka1aXcJ7VOTNcZfePDKYbyxbg/fnr+OubeNI/G7LiLh8fMqz8uBfHcvCCoY+dC2fUe5e04u53ZP45c3jSJZM2aKSAQ1e4Hn7luBkc39uU0lFnPe2bKPhZuKGNwzXdMvC13T2vK1K4byrflrWbCqkCkX1XYvYxFpLczsXKCnu/+z2vKPm1lbd/8goNAEOHT8JF96ZhnJScZTt2SRpuugRSSiNJ//R3TwWBlPvr2VTz76D2757VKOlJZz16Rzgg5LWojpY/szol9HfvTqBkpKTwYdjog0jl8Cpxt7fTzxngSkvCLGfc+uYEfxMX4zYzQZXdoHHZKISGB0eKueVu04yKzsfF5eVciJ8hhjBnTm3z41hCuH96JtSnLQ4UkLkZxk/GDKcK79r0X88s3NfPsz5wcdkog03AB3X119obvnmNmA5g9HANydh19ex9vv7+Unn72Qced0DTokEZFAqcCrg+NlFby8upDZ2fmsLjhE+9Rkrh/djxnjMzmv99lBhyct1MiMTkwf25/fv5fHtKx+DOul3xWRVq5dLe/pAuyA/Oebm0/NmHnjGN2qSEREBV4ttu07ypzsfJ7PLeDQ8ZMM7pHG96dcwNRRfUlv1ybo8KQV+PoVQ/nLml18e/5anrtjgiZcEWndlpnZbe7+ZNWFZvYlIDegmCLtd4u28djfN3NDVj8eulIzZoqIgAq80zp4rIyv/GklCzftJSXJuGJ4L2aOz2TcwC76A13qpVP7VB66ahgP/nkNP3p1A7dPOoce6bWdBAiH8ooYf99YxBvrdnPLhAGMzOgUdEgijeEB4rNAf54PC7osIBWYGlhUETV/xU6+9/J6Lj+/Jz+eeqHys4hIggq8aipizn3PriB7636+MnkI08dm0OPs8P9BLk1n2ugMFm3Zz1PvbuP37+VxZeKAwdgQHjAoKinlT0t3MHfpdnYdKiXJ4NXVu/jp9SM0m6i0eu6+B/iYmX0SGJ5Y/Kq7vxVgWJG0cGMRX31+FePP6cJj00eRkqw540REKqnAq+anb2zknc37eOS6C7lprMbyS8MlJRmPTR/FA5MHM2fJdp7P2cErq3cxpGcaM8dncm0rH/Lr7izdVsys7HxeX7ub8pjz8cHdePiaCxiV0Yl75i7ny39cyZaiI3xl8hDdSkRaPXdfCCwMOo6oyskr5q45uQzrnc6TN2fRro0mOBMRqcrcPegYzigrK8tzcnKa/HMWrCrk/mdXMGN8f3547YVN/nkSTcfLKnh5VSGzsvNZs/MQHVKTmXpxX2aMz2xVE7GUlJ5k/oqdzMrO5/09Rzi7XQo3ZGXw+fGZDOzW4dR6ZeUxvj1/LX/K2cEVF/TkFzdcRAfdn0pqYGa57p4VdBytRXPlx5Ziw67D3Pj4YrqlteW5OyfQLa1t0CGJiDSbuuZIFXgJa3ce4vrfvMeIvp2Yfes4UlM03EOaXuVtNxasKqSsPMbYAV2YMSGTKy/o1WJ/BzfuPszs7HxeXL6To2UVXNi3IzMnZHL1iD6clXr6I+nuzu8W5fHDV9czpGc6T92SRb/Ouk+V/E8q8OonSgXe9v3H+Oxv3iPZjHl3TdB3iIhEjgq8eth/5ATX/L9FxNxZcO8ldE/XEUFpXgeOlvF87g5mZ29ne/ExuqWlcuOYDD43LpO+nRp/9vU9h0uZl1vArkPH67Xdpt0lLMs7QNuUJK4e2YeZ4zPrNYHKP9/fy71zl5OanMTjM0eTNaBLfUOXkFOBVz9RKfCKSkqZ9pvFHDp+kufvmMDgnulBhyQi0uxU4NXRyYoYNz+9lNztB5h35wRG9NNsfxKcWMx5e/NeZmfn89bGIgAuG9aTmRMy+fi53Rp0/Zq7s3jrfmZn5/PGuj1UxJwuHVKpT4tdOqRyQ1YG14/uR+cOqR8pji1FR7j1mWXsPHicH0+9kGlZGR+pHQknFXj1E4UC79Dxk9z0RDb5+48y59ZxjOrfOeiQREQCUdccGfkLYX782gYWb93Po9NGqriTwCUlGZcO7cGlQ3tQcOAYzy7dzh+X7uDNDXvI7NqeGeMymZbVj07t615cHS49yQu5Bcxesp0tRUfoeFYbvjhxAJ8fl8mAKtfKNZdze6Qx/56J3DN3OV+bt5r395Tw0FXnkazJV0SkmuNlFdz6zDK2FJXw9C1jVNyJiNRBpM/gzcst4KvPr+KLEwfynavPb/T2RRrDifIKXl+7m9nZ+fUaHrm+8DCzsvN5aeVOjpVVMDKjEzPG9efqkX1axKxzJyti/PCV9TyzOJ9Lh3bnsemjOLsVzyYqjUNn8OonzGfwTlbEuHNWLm9tKuJX00fxmRF9gg5JRCRQOoN3Bit3HOSbL67hY4O68s1PDws6HJEatU1JZspFfZlyUV827EpMcLJiJ/NyCxjRryMzxmdyTaJoqywGZy3OJyc/XgxOuagPM8Zntrgz1G2Sk/jelOEM6ZXOd19ax3X/9R5P3ZwVyFlFEWlZYjHn6/NW8/eNRfzw2uEq7kRE6iGSZ/CKSkq55leLSEk2Ftx7CV0+4rVEIkEpKT3Jiyt2MmtxPpsTwy4/ObQ772zex/6jZQzo2p4Z4zO5fnT9hnMGZfEH+7lrTi7u8OOpF3LFBT2b5MbFe0tO8OaGPYzs14nz+7SeW1K0dLGY88/Neyk4cJyZ4zMb1JbO4NVP2M7gHTlRTm7+AV5YXsBLKwv5908N4b7/NTjosEREWgSdwatBWXmMu2cv5+DxMl64a6KKO2mV0tu14eYJA5g5PpMliZuM/3X9Hi45txszJ2QycVDDJmRpbhMGdeWleyZy2x9yuGfucvp0bMfnxvXnxjH9GzyrrbuzLO9A4kbsuzhZET+oNTqzMzPHZ3LVhb1omxL8kNXWqPhoGc/n7GD2knx2FB8ns2t7Pje2v66nlDorPlrGsrxilm6LP9YVHiLmkJxk3DlpEPdedm7QIYqItDqRO4P3zRfXMHfJdn41fRRXj9SQD5GWpLwixpsb9jArO59FW/bTJtm4cnhvZo7PZMyAzpjVvXA4cqKcF1fsZPbifDbtKSG9XQrTRmcwdVRfluYVMzs7n237jtKlQ+KWFGP7k9FF99U6E3dnxY6DzM7O55XVuygrjzFuYBdmTsjk8vMbfv9GncGrn9Z2Bm/3oVKWbNvP0m3FLMsr5v09RwBITUliVEYnxg3swpiBXbi4f2c6tI3cMWgRkVq16NskmNmVwP8FkoGn3P2R2tZvrAQ2Z0k+//HiWu66dBAPXqnr7kRasg/2HmFO9naez91BSWk5Q3umM2NCJlNH9SWtlj/8Nu0uYXZ2Pi8sL+BoWQUX9Dmbmydkcs3Ivv9yI/ZYzHnvg/3Mys7jb+v34MBlQ3swY0ImkwZ3b1VnQJvDsbJyFqwsZFZ2PusKD5PWNoXrLu7LjPGZDGnEe5KFucA7U+4zs7bAH4DRwH7gRnfPq63NllzgHThaxsbdJWzYdZh1hYdZllfM9uJjAKS1TWF0ZmfGDuzCuIFduLBfR51JFxE5gxZb4JlZMvA+8CmgAFgGTHf39TVt0xgJLCevmOlPZvOxQd347RfGaAiRSCtxrKycl1cV8ofF8cKiQ2oy113cjxnjMxnaK15YlJXHeH3dbmYvzmdpXjGpKUl8ZkT8zN9FGZ3OeOav8OBx/rh0O3OX7mDfkRNkdDmLz4/L5IasjMgP4/5g7xFmZ+czL7eAktJyhvVKZ8b4TK49Q6H9UYW1wKtL7jOzu4ER7n6nmd0ETHX3G2trtyUUeOUVMbbuO8qGXYdPFXQbd5Ww+3DpqXW6paUmCrqujB3QhfN6pzfJdbYiImHWkgu8CcDD7n5F4vU3ANz9/9S0TUMT2O5DpXzmV++S1jaZl+65hI7tNRW7SGvj7qzccZBZVYYGjh3YhRF9OzJ/ZeGpwix+r8CPVpiVlcd4Y91uZmXns3RbolC8sDcfO7cbUTsmVHoyxqtrCk8Nlb1qeG9mTsgkK7N+Q2XrK8QF3hlzn5m9kVhnsZmlALuB7l5Lom6MAu+llTupiNXvb4Hio2Vs2FXCxt2H2Vx0hLLyGABtko1B3dM4r/fZnNc7nWG9zmZY73R6pLdrUIwiItKyJ1npC+yo8roAGFd9JTO7HbgdoH///g36wCfe3srxsnLm3jZOxZ1IK2VmjOrfmVH9O/Ot/33+qck9luUVN9rQytTEPQavHtmHTbtLmLMknxeW7+SFFTsbsSetR5+O7fjaFUO5ISujwZPdSJ1y36l13L3czA4BXYF9VVdqzPwI8OCfV1N6Mlbv7bqnt+W83mdzybndGJYo5gZ1T2vwdZgiItIwQZzBmwZc4e63Jl7PBMa6+301bdPQI5TlFTE27Snhgj4dP3IbItLyxGLOkbLyJr1B+rGycvaWnGiy9lsqw+jTqV2zD6ML8Rm8M+Y+M1uXWKcg8fqDxDr7a2q3Mc7gbd9/DKd+fwuktU2ha5qKfhGR5tSSz+AVABlVXvcDCpvyA1OSk1TciYRQUpI1aXEH0D41hcyums1PGqwuua9ynYLEEM2OQHFTB9a/q2aPFREJkyDGUSwDBpvZQDNLBW4CFgQQh4iISHOpS+5bANySeH498FZt19+JiIicTrMflk5cV3Av8AbxqaJ/6+7rmjsOERGR5lJT7jOz7wM57r4AeBqYZWZbiJ+5uym4iEVEpLUKZNyRu78GvBbEZ4uIiAThdLnP3b9T5XkpMK254xIRkXDRVFciIiIiIiIhoQJPREREREQkJJr9NgkfhZntBfIb2Ew3qt1LKCLU72iJYr+j2GcIb78z3b170EG0Fo2UHyG8v09nEsV+R7HPoH5HTVj7Xacc2SoKvMZgZjlhvLfSmajf0RLFfkexzxDdfkvTiOrvUxT7HcU+g/oddBzNLar9rqQhmiIiIiIiIiGhAk9ERERERCQkolTgPRF0AAFRv6Mliv2OYp8huv2WphHV36co9juKfQb1O2qi2m8gQtfgiYiIiIiIhF2UzuCJiIiIiIiEWiQKPDO70sw2mdkWM3so6Hiai5nlmdkaM1tpZjlBx9NUzOy3ZlZkZmurLOtiZn8zs82Jn52DjLGx1dDnh81sZ2J/rzSzTwcZY1MwswwzW2hmG8xsnZl9ObE8tPu7lj6Hfn9L01N+VH4M0/dlpSjmyCjmR1COrEnoh2iaWTLwPvApoABYBkx39/WBBtYMzCwPyHL3MN4H5BQz+wRwBPiDuw9PLPspUOzujyT+aOns7g8GGWdjqqHPDwNH3P3nQcbWlMysN9Db3ZebWTqQC1wLfIGQ7u9a+nwDId/f0rSUH5Ufw/Z9WSmKOTKK+RGUI2sShTN4Y4Et7r7V3cuAPwJTAo5JGpG7vw0UV1s8BXgm8fwZ4v/ZQ6OGPoeeu+9y9+WJ5yXABqAvId7ftfRZpKGUH0MuivkRopkjo5gfQTmyJlEo8PoCO6q8LiA6O96Bv5pZrpndHnQwzaynu++C+H9+oEfA8TSXe81sdWJ4SqiGYVRnZgOAUcASIrK/q/UZIrS/pUkoPyo/hvb7sgaR+M6MYn4E5ciqolDg2WmWhXtc6ocmuvvFwFXAPYkhCxJe/x8YBFwE7AIeDTacpmNmacCfgQfc/XDQ8TSH0/Q5Mvtbmozyo/JjlETiOzOK+RGUI6uLQoFXAGRUed0PKAwolmbl7oWJn0XAi8SH40TFnsS47Mrx2UUBx9Pk3H2Pu1e4ewx4kpDubzNrQ/xLfI67v5BYHOr9fbo+R2V/S5NSflR+DN33ZU2i8J0ZxfwIypGnE4UCbxkw2MwGmlkqcBOwIOCYmpyZdUhcbIqZdQAuB9bWvlWoLABuSTy/BXgpwFiaReUXeMJUQri/zcyAp4EN7v6LKm+Fdn/X1Oco7G9pcsqPyo+h+r6sTdi/M6OYH0E5siahn0UTIDE16i+BZOC37v6jgENqcmZ2DvGjkgApwNyw9tvMngUuBboBe4DvAvOB54D+wHZgmruH5oLrGvp8KfGhCA7kAXdUjrsPCzO7BHgHWAPEEou/SXy8fSj3dy19nk7I97c0PeVH5UdC9H1ZKYo5Mor5EZQjaxKJAk9ERERERCQKojBEU0REREREJBJU4ImIiIiIiISECjwREREREZGQUIEnIiIiIiISEirwREREREREQkIFnkSGmbmZPVrl9VfN7OFGavv3ZnZ9Y7R1hs+ZZmYbzGxhA9r4ZrXX7zU8MhERaa2UH0+1ofwooaACT6LkBHCdmXULOpCqzCy5Hqt/Cbjb3T/ZgPb+JYG5+8fq8fkiIhI+yo9xyo8SCirwJErKgSeAr1R/o/oRRjM7kvh5qZn908yeM7P3zewRM/u8mS01szVmNqhKM5PN7J3Eep9JbJ9sZj8zs2VmttrM7qjS7kIzm0v85pzV45meaH+tmf0ksew7wCXAb8zsZ9XW/x/tmdl8M8s1s3Vmdnti2SPAWWa20szmnKav/zCzeWa20czmmJkl3vt0Ytm7ZvaYmb2SWD4p0dZKM1thZun13y0iIhIw5UflRwmRlKADEGlmvwZWm9lP67HNSOA8oBjYCjzl7mPN7MvAfcADifUGAJOAQcBCMzsXuBk45O5jzKwtsMjM/ppYfyww3N23Vf0wM+sD/AQYDRwA/mpm17r7983sMuCr7p5zmjirt/dFdy82s7OAZWb2Z3d/yMzudfeLaujrKOACoBBYBEw0sxzgceAT7r7NzJ6tsv5XgXvcfZGZpQGltf5LiohIS6X8qPwoIaEzeBIp7n4Y+ANwfz02W+buu9z9BPABUJmA1hBPWpWec/eYu28mnuiGAZcDN5vZSmAJ0BUYnFh/afXklTAG+Ie773X3cmAO8Ik6xFm9vfvNbBWQDWRU+dwztVHg7jFgZaJ/w4CtVdqumsAWAb8ws/uBTol4RUSklVF+rFMbyo/SKqjAkyj6JfGx+h2qLCsn8f8hMewitcp7J6o8j1V5HeNfz4J7tc9xwID73P2ixGOgu1cmwKM1xGd17Ug1p9ozs0uBycAEdx8JrADa1aGNqn2tIN6/GuNx90eAW4GzgGwzG1b/sEVEpIVQfqyZ8qO0GirwJHLcvRh4jngSq5RHfMgHwBSgzUdoepqZJSWuOzgH2AS8AdxlZm0AzGyImXWorRHiRzInmVk3i18QPh34Zz1j6QgccPdjiaQyvsp7JyvjqaONwDlmNiDx+sbKN8xskLuvcfefADnEj2aKiEgrpPyo/CjhoAJPoupRoOpsYU8STxpLgXHUfPSwNpuIJ5q/AHe6eynwFLAeWG5ma4mP1a/12ld33wV8A1gIrAKWu/tL9YzldSDFzFYDPyA+DKXSE8Svs5hTl4bc/ThwN/C6mb0L7AEOJd5+IHGh+yrgOPG+i4hI66X8qPworZy5Vz9rLiLyr8wszd2PJIbn/BrY7O7/GXRcIiIiQVJ+lJZIZ/BEpC5uS1wIv4748JbHA45HRESkJVB+lBZHZ/BERERERERCQmfwREREREREQkIFnoiIiIiISEiowBMREREREQkJFXgiIiIiIiIhoQJPREREREQkJFTgiYiIiIiIhMR/A97IiIbw7t+MAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x1a12acb978>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "\n",
    "item_nbr_ratings = actual_ratings.apply(lambda col: np.sum(~np.isnan(col)), axis=1)\n",
    "item_max_nbr_ratings = item_nbr_ratings.max()\n",
    "range_item_max_nbr_ratings = range(item_max_nbr_ratings+1)\n",
    "\n",
    "plt.figure(figsize=(15,3))\n",
    "plt.subplot(121)\n",
    "nbr_ratings_items = []\n",
    "for i in range_item_max_nbr_ratings:\n",
    "    nbr_ratings_items.append(len(item_nbr_ratings[item_nbr_ratings == i]))\n",
    "plt.plot(nbr_ratings_items)\n",
    "plt.xlabel('Number of ratings')\n",
    "plt.ylabel('Amount of items')\n",
    "plt.title('Histogram of amount of ratings')\n",
    "\n",
    "plt.subplot(122)\n",
    "cum_nbr_ratings_items = []\n",
    "for i in range(len(nbr_ratings_items)):\n",
    "    cum_nbr_ratings_items.append(np.sum(nbr_ratings_items[:i]))\n",
    "    \n",
    "cum_nbr_ratings_items = np.array(cum_nbr_ratings_items)\n",
    "plt.plot(cum_nbr_ratings_items/actual_ratings.shape[0])\n",
    "plt.xlabel('Number of ratings')\n",
    "plt.ylabel('Cumulative distribution')\n",
    "plt.title('Cumulative histogram of amount of ratings');"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}