tests

Author: remstef

pom.xml

@@ -40,6 +40,7 @@
 			<plugin>
 				<groupId>org.scala-tools</groupId>
 				<artifactId>maven-scala-plugin</artifactId>
+				<version>2.15.2</version>
 				<executions>
 					<execution>
 						<goals>
@@ -81,6 +82,12 @@
 			<artifactId>spark-core_2.10</artifactId>
 			<version>1.2.0-cdh5.3.2</version>
 		</dependency>
+		<dependency>
+			<groupId>junit</groupId>
+			<artifactId>junit</artifactId>
+			<version>4.12</version>
+			<scope>test</scope>
+		</dependency>
 	</dependencies>
 
 </project>

src/main/java/sparkfun/SparkConfigured.java

@@ -0,0 +1,54 @@
+/*
+ *   Copyright 2015
+ *
+ *   Licensed under the Apache License, Version 2.0 (the "License");
+ *   you may not use this file except in compliance with the License.
+ *   You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *   Unless required by applicable law or agreed to in writing, software
+ *   distributed under the License is distributed on an "AS IS" BASIS,
+ *   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *   See the License for the specific language governing permissions and
+ *   limitations under the License.
+ */
+package sparkfun;
+
+import org.apache.spark.SparkConf;
+
+/**
+ * @author Steffen Remus
+ *
+ */
+public abstract class SparkConfigured {
+	
+	public abstract void run(SparkConf conf, String[] args);
+
+	/*
+	 * -key1=val1 --key2=val2 -key3 val3 -key4 val4 val5 -key6=val7 val8 -key9 
+	 * -key1=val1 --key2=val2 -key3 val3 -key4 val5 val6
+	 */
+	public void run(String[] args){
+		SparkConf conf = new SparkConf();
+		String key = "", value = "";
+		for(int i = 0; i < args.length; i++){
+			if(args[i].startsWith("-")){
+				key = args[i].replaceFirst("--?", ""); // match -key, --key
+				value = "";
+			}
+			else
+				value += " " + args[i];
+			if(key.contains("=")){
+				value = key.substring(key.indexOf("=")+1);
+				key = key.substring(0, key.indexOf("="));
+				conf.set(key.trim(), value.trim());
+				continue;
+			}
+			if(!key.isEmpty() && !value.isEmpty())
+				conf.set(key.trim(), value.trim());
+		}
+		run(conf, args);
+	}
+	
+}

src/main/java/sparkfun/TestParams.java

@@ -0,0 +1,58 @@
+/*
+ *   Copyright 2015
+ *
+ *   Licensed under the Apache License, Version 2.0 (the "License");
+ *   you may not use this file except in compliance with the License.
+ *   You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *   Unless required by applicable law or agreed to in writing, software
+ *   distributed under the License is distributed on an "AS IS" BASIS,
+ *   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *   See the License for the specific language governing permissions and
+ *   limitations under the License.
+ */
+package sparkfun;
+
+import org.apache.spark.SparkConf;
+import org.apache.spark.api.java.JavaSparkContext;
+
+import scala.Tuple2;
+
+
+/**
+ * @author Steffen Remus
+ *
+ */
+public class TestParams extends SparkConfigured{
+	
+	public void run(SparkConf conf, String[] args) {
+		
+		conf.setAppName("TestParams");
+	
+		JavaSparkContext ctx = new JavaSparkContext(conf);
+		
+		System.out.println(ctx.appName());
+		System.out.println(ctx.getConf().getOption("testopt"));
+		
+		for(Tuple2<?,?> t : ctx.getConf().getAll())
+			System.out.println(t);
+		System.out.println(ctx.getConf().getOption("spark.master"));
+		
+		try {
+			Thread.sleep(10000);
+		} catch (InterruptedException e) {
+			// TODO Auto-generated catch block
+			e.printStackTrace();
+		}
+			    
+		ctx.close();
+
+	}
+	
+	public static void main(String[] args) {
+		new TestParams().run(args);
+	}
+
+}

src/test/java/sparkfun/SparkConfiguredTest.java

@@ -0,0 +1,47 @@
+/*
+ *   Copyright 2015
+ *
+ *   Licensed under the Apache License, Version 2.0 (the "License");
+ *   you may not use this file except in compliance with the License.
+ *   You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *   Unless required by applicable law or agreed to in writing, software
+ *   distributed under the License is distributed on an "AS IS" BASIS,
+ *   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *   See the License for the specific language governing permissions and
+ *   limitations under the License.
+ */
+package sparkfun;
+
+import java.util.Arrays;
+
+import org.apache.spark.SparkConf;
+import org.junit.Test;
+
+import scala.Tuple2;
+
+/**
+ * @author Steffen Remus
+ *
+ */
+public class SparkConfiguredTest {
+
+	@Test
+	public void test() {
+		SparkConfigured job = new SparkConfigured() {
+			@Override
+			public void run(SparkConf conf, String[] args) {
+				System.out.println(Arrays.toString(args));
+				for(Tuple2<String, String> t : conf.getAll())
+					System.out.println(t);
+			}
+		};
+		job.run("-key1=val1 --key2=val2 -key3 val3 -key4 val4.1 val4.2 -key5=val5.1 val5.2 -key6".split(" "));
+		job.run("-key1=val1 --key2=val2 -key3 val3 -key4 val4.1 val4.2".split(" "));
+		job.run("---key1=val1".split(" "));
+
+	}
+
+}