Merge pull request #13 from rajdeepd/master

changes in chapter 4 scala code - added package

Author: badlogicmanpreet

Chapter_04/python/code04.ipynb

@@ -1,7 +1,13 @@
-name := "code03"
+name := "chapter04"
 
 version := "1.0"
 
 scalaVersion := "2.10.4"
 
 libraryDependencies += "org.apache.spark" %% "spark-core" % "1.6.0"
+libraryDependencies += "org.scalanlp" %% "breeze" % "0.12"
+// native libraries are not included by default. add this if you want them (as of 0.7)
+// native libraries greatly improve performance, but increase jar sizes.
+// It also packages various blas implementations, which have licenses that may or may not
+// be compatible with the Apache License. No GPL code, as best I know.
+libraryDependencies +="org.scalanlp" %% "breeze-natives" % "0.12"

Chapter_04/scala/build.sbt

@@ -1,3 +1,5 @@
+package org.sparksamples
+
 /**
   * Created by Rajdeep on 12/22/15.
   */

Chapter_04/scala/src/main/scala/MovieData.scala renamed to Chapter_04/scala/src/main/scala/org/sparksamples/MovieData.scala

@@ -1,12 +1,19 @@
+package org.sparksamples
+
 /**
   * Created by Rajdeep on 12/22/15.
   */
 import org.apache.spark.SparkContext
+import breeze.linalg.DenseVector
 
 object RatingData {
+  val util = new Util()
+  val sc = new SparkContext("local[2]", "First Spark App")
+  util.sc = sc
+  val user_data = util.getUserData()
 
   def main(args: Array[String]) {
-    val sc = new SparkContext("local[2]", "First Spark App")
+
     val rating_data_raw = sc.textFile("../../data/ml-100k/u.data")
 
     println(rating_data_raw.first())
@@ -28,9 +35,9 @@ object RatingData {
     println("min_rating: " + min_rating)
     println("mean_rating: " + mean_rating)
 
-    var user_data = sc.textFile("../../data/ml-100k/u.user")
-    user_data = user_data.map(l => l.replaceAll("[|]", ","))
-    val user_fields = user_data.map(l => l.split(","))
+    println("user_data.first():"  + user_data.first())
+    val user_fields = user_data.map(l => l.split("\\|"))
+    //var num_users = util.getUserFields()
     val num_users = user_fields.map(l => l(0)).count()
     //val median_rating = math.median(ratings.collect()) function not supported - TODO
     val ratings_per_user = num_ratings / num_users
@@ -70,7 +77,21 @@ object RatingData {
 
     println("Encoding of 'doctor : " + all_occupations_dict("doctor"))
     println("Encoding of 'programmer' : " + all_occupations_dict("programmer"))
-
+    //println(all_occupations_dict)
+    /*
+    K = len(all_occupations_dict)
+    binary_x = np.zeros(K)
+    k_programmer = all_occupations_dict['programmer']
+    binary_x[k_programmer] = 1
+    print "Binary feature vector: %s" % binary_x
+    print "Length of binary vector: %d" % K
+     */
+    val k = all_occupations_dict.size
+    val binary_x = DenseVector.zeros[Double](k)
+    val k_programmer = all_occupations_dict("programmer")
+    binary_x(k_programmer) = 1
+    println("Binary feature vector: %s" + binary_x)
+    println("Length of binary vector: " + k)
     sc.stop()
   }
 

Chapter_04/scala/src/main/scala/RatingData.scala renamed to Chapter_04/scala/src/main/scala/org/sparksamples/RatingData.scala

@@ -1,3 +1,5 @@
+package org.sparksamples
+
 /**
   * Created by Rajdeep on 12/22/15.
   */

Chapter_04/scala/src/main/scala/UserData.scala renamed to Chapter_04/scala/src/main/scala/org/sparksamples/UserData.scala

@@ -1,3 +1,5 @@
+package org.sparksamples
+
 import org.apache.spark.SparkContext
 import org.apache.spark.rdd.RDD
 
@@ -8,7 +10,6 @@ class Util {
   var sc:SparkContext = null
 
   def getMovieDataRDD(): RDD[String] = {
-
     val movie_data = sc.textFile("../../data/ml-100k/u.item")
     return movie_data
   }
@@ -25,5 +26,17 @@ class Util {
     return x.sum/x.length
   }
 
+  def getUserData() : RDD[String] = {
+    //val sc = new SparkContext("local[2]", "First Spark App")
+    var user_data = sc.textFile("../../data/ml-100k/u.user")
+    return user_data
+  }
+
+  def getUserFields() : RDD[Array[String]] = {
+    val user_data = this.getUserData()
+    val user_fields = user_data.map(l => l.split(","))
+    return user_fields
+  }
+
 
 }