up

Author: remstef

.gitignore

@@ -11,6 +11,10 @@ local_data
 .settings
 .cache
 
+# intellij related
+.idea
+*.iml
+
 # user defined ignore
 **/*_ignore
 

pom.xml

@@ -82,6 +82,13 @@
 			<artifactId>spark-core_2.10</artifactId>
 			<version>1.2.0-cdh5.3.2</version>
 		</dependency>
+		
+		<dependency>
+		  <groupId>org.scalanlp</groupId>
+		    <artifactId>breeze_2.10</artifactId>
+		    <version>0.10</version>
+		</dependency>
+
 		<dependency>
 			<groupId>junit</groupId>
 			<artifactId>junit</artifactId>

src/main/java/sparkfun/breezetest/test.scala

@@ -0,0 +1,36 @@
+/*
+ *   Copyright 2015
+ *
+ *   Licensed under the Apache License, Version 2.0 (the "License");
+ *   you may not use this file except in compliance with the License.
+ *   You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *   Unless required by applicable law or agreed to in writing, software
+ *   distributed under the License is distributed on an "AS IS" BASIS,
+ *   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *   See the License for the specific language governing permissions and
+ *   limitations under the License.
+ */
+package sparkfun.breezetest
+
+import breeze.linalg.{sum, DenseVector}
+
+/**
+ * @author Steffen Remus
+ *
+ */
+object test {
+  
+  def main(args: Array[String]): Unit = {
+    val s = DenseVector(0,1,2,3);
+    val t = s + DenseVector(1,1,1,1);
+    val z = t.foldLeft("")((x,y) => x + "\t" + y.toString());
+
+    println(sum(t(0,2)))
+    println(sum(t))
+    println("a b c".split(" ", 5).toList)
+  }
+
+}

src/main/java/sparkfun/wc/AppJ8.java

@@ -38,7 +38,8 @@ public static void main(String[] args) {
 			System.exit(1);
 		}
 
-		SparkConf sparkConf = new SparkConf().setAppName("JavaWordCountJ8");
+		SparkConf sparkConf = new SparkConf().setAppName("JavaWordCountJ8").setMaster("local[2]").set("spark.io.compression.codec","org.apache.spark.io.LZ4CompressionCodec");
+		//sparkConf.set("spark.serializer", "org.apache.spark.serializer.KryoSerializer");
 		JavaSparkContext ctx = new JavaSparkContext(sparkConf); // "local[1]","test",
 		JavaRDD<String> lines = ctx.textFile(args[0], 1);
 

src/main/java/sparkfun/wc/AppSc.scala

@@ -18,7 +18,7 @@ object AppSc {
 		}
 
 
-		val conf = new SparkConf().setAppName("ScalaWordCount");
+		val conf = new SparkConf().setAppName("ScalaWordCount").setMaster("local[2]").set("spark.io.compression.codec","org.apache.spark.io.LZ4CompressionCodec");
 		val sc = new SparkContext(conf);
 
 		val lines = sc.textFile(args(0));

src/test/files/artifical-ct.txt

@@ -0,0 +1,6 @@
+1	a	b	3	1	0	6
+1	a	c	4	0	6	0
+1	b	c	3	0	7	0
+2	b	c	2	1	0	12	
+2	b	d	1	2	9	3	
+2	c	d	5	2	5	3

src/test/files/artifical-test.txt

@@ -0,0 +1,11 @@
+a	b	5
+a	c	25
+b	a	735
+b	c	8
+b	d	4345
+b	e	73
+c	d	253
+c	e	13
+d	a	5345
+d	b	45
+d	c	176