Movie_rating_count.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from pyspark.sql import SparkSession\n",
    "import collections\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "spark = SparkSession.builder.appName(\"CountMovieRatings\").master(\"local[2]\").getOrCreate()\n",
    "data = spark.sparkContext.textFile(\"/home/maniram/big_data/movie/u.data\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['1,31,2.5,1260759144',\n",
       " '1,1029,3.0,1260759179',\n",
       " '1,1061,3.0,1260759182',\n",
       " '1,1129,2.0,1260759185',\n",
       " '1,1172,4.0,1260759205']"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.take(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "\n",
    "movieId = '380'   # Movie Id \n",
    "ratingColId = 2   #  Column Number of Rating\n",
    "movieColId = 1    #  Column number for movie id\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Extract Rating Record for Movie id 380"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[['4', '380', '3.0'],\n",
       " ['7', '380', '4.0'],\n",
       " ['15', '380', '4.0'],\n",
       " ['19', '380', '3.0'],\n",
       " ['20', '380', '4.0'],\n",
       " ['21', '380', '3.0'],\n",
       " ['23', '380', '4.0'],\n",
       " ['24', '380', '4.0'],\n",
       " ['26', '380', '3.0'],\n",
       " ['30', '380', '4.0']]"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "movie_data = data.map(lambda x: x.split(',')[0:3]).filter(lambda moviecol:moviecol[movieColId]==movieId)\n",
    "\n",
    "movie_data.take(10)\n",
    "# movie_data = movie_data.filter(lambda x:x[movieColId]==movieId)\n",
    "# movie_data.take(20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "countHisto = movie_data.map(lambda row:(row[ratingColId],1))\\\n",
    "  .reduceByKey(lambda x,y:x+y)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Rating Counts For movie"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('0.5', 1),\n",
       " ('1.0', 2),\n",
       " ('1.5', 1),\n",
       " ('2.0', 17),\n",
       " ('2.5', 9),\n",
       " ('3.0', 54),\n",
       " ('3.5', 20),\n",
       " ('4.0', 63),\n",
       " ('4.5', 7),\n",
       " ('5.0', 24)]"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "countHisto.sortByKey().collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}