[ENV-73] Move to Maven modules (#119)

* First stab at modularization. Not compiling yet...

* Much closer to compiling now

* Remove metastore objects

* Remove more metastore junk

* Update gitignore for Eclipse and Mac junk

* Remove Eclipse junk

* Remove Mac junk

* Move test resources to correct modules

* Bunch of pom work

* Apply changes not carried over from rebase

* Shade Guava, ignore local metastore files

* Build poms

* Re-add license header check

* Refactor package names for external modules

* Minor touchups

* Update readme and pom

* Add inception year back for license header

* Bring Guava version back to what we had prior

* Tabs to spaces

* Use project version for uber jar dependencies

.gitignore

@@ -27,3 +27,17 @@ buildNumber.properties
 .idea
 *.iml
 /todo
+
+#Eclipse
+.classpath
+.settings
+.settings/
+.project
+
+#Mac
+.DS_Store
+
+#Local Hive Metastore
+metastore_db/
+spark-warehouse/
+derby.log

README.md

@@ -19,15 +19,15 @@ Envelope requires a CDH5.7+ cluster with:
 
 - Cloudera's distribution of Apache Spark 2.1.0, or above
 - Cloudera's distribution of Apache Kafka 2.1.0 (based on Apache Kafka 0.10) or above, if using that component
-- Cloudera's distribution of Apache Kudu 1.3.0, if using that component
+- Cloudera's distribution of Apache Kudu 1.3.0 or above, if using that component
 
 ### Compiling Envelope
 
 You can build the Envelope application from the top-level directory of the source code by running the Maven command:
 
     mvn clean package
 
-This will create `envelope-0.5.0.jar` in the target directory.
+This will create `envelope-0.5.0.jar` in the `build/envelope/target` directory.
 
 ### Finding examples
 

build/envelope/pom.xml

@@ -0,0 +1,48 @@
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+  <modelVersion>4.0.0</modelVersion>
+  <parent>
+    <groupId>com.cloudera.labs.envelope</groupId>
+    <artifactId>envelope-parent</artifactId>
+    <version>0.5.0-SNAPSHOT</version>
+    <relativePath>../../pom.xml</relativePath>
+  </parent>
+  <artifactId>envelope</artifactId>
+
+  <dependencies>
+    <dependency>
+      <groupId>com.cloudera.labs.envelope</groupId>
+      <artifactId>envelope-core</artifactId>
+      <version>${project.version}</version>
+    </dependency>
+    <dependency>
+      <groupId>com.cloudera.labs.envelope</groupId>
+      <artifactId>envelope-examples</artifactId>
+      <version>${project.version}</version>
+    </dependency>
+    <dependency>
+      <groupId>com.cloudera.labs.envelope</groupId>
+      <artifactId>envelope-hbase</artifactId>
+      <version>${project.version}</version>
+    </dependency>
+    <dependency>
+      <groupId>com.cloudera.labs.envelope</groupId>
+      <artifactId>envelope-kafka</artifactId>
+      <version>${project.version}</version>
+    </dependency>
+    <dependency>
+      <groupId>com.cloudera.labs.envelope</groupId>
+      <artifactId>envelope-kudu</artifactId>
+      <version>${project.version}</version>
+    </dependency>
+    <dependency>
+      <groupId>com.cloudera.labs.envelope</groupId>
+      <artifactId>envelope-lib</artifactId>
+      <version>${project.version}</version>
+    </dependency>
+    <dependency>
+      <groupId>com.cloudera.labs.envelope</groupId>
+      <artifactId>envelope-zookeeper</artifactId>
+      <version>${project.version}</version>
+    </dependency>
+  </dependencies>
+</project>

core/pom.xml

@@ -0,0 +1,45 @@
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+  <modelVersion>4.0.0</modelVersion>
+  <parent>
+    <groupId>com.cloudera.labs.envelope</groupId>
+    <artifactId>envelope-parent</artifactId>
+    <version>0.5.0-SNAPSHOT</version>
+    <relativePath>../pom.xml</relativePath>
+  </parent>
+  <artifactId>envelope-core</artifactId>
+
+  <dependencies>
+    <dependency>
+      <groupId>com.google.guava</groupId>
+      <artifactId>guava</artifactId>
+      <version>${guava.version}</version>
+      <scope>compile</scope>
+    </dependency>
+  </dependencies>
+
+  <build>
+    <plugins>
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-shade-plugin</artifactId>
+        <version>2.4</version>
+        <executions>
+          <execution>
+            <phase>package</phase>
+            <goals>
+              <goal>shade</goal>
+            </goals>
+            <configuration>
+              <relocations>
+                <relocation>
+                  <pattern>com.google.common</pattern>
+                  <shadedPattern>envelope.shaded.com.google.common</shadedPattern>
+                </relocation>
+              </relocations>
+            </configuration>
+          </execution>
+        </executions>
+      </plugin>
+    </plugins>
+  </build>
+</project>

src/main/java/com/cloudera/labs/envelope/derive/DeriverFactory.java → core/src/main/java/com/cloudera/labs/envelope/derive/DeriverFactory.java

@@ -30,39 +30,42 @@ public static Deriver create(Config config) {
 
     String deriverType = config.getString(TYPE_CONFIG_NAME);
 
+    String deriverClass;
     Deriver deriver;
 
     switch (deriverType) {
       case "sql":
-        deriver = new SQLDeriver();
+        deriverClass = "com.cloudera.labs.envelope.deriver.SQLDeriver";
         break;
       case "passthrough":
-        deriver = new PassthroughDeriver();
+        deriverClass = "com.cloudera.labs.envelope.deriver.PassthroughDeriver";
         break;
       case "nest":
-        deriver = new NestDeriver();
+        deriverClass = "com.cloudera.labs.envelope.deriver.NestDeriver";
         break;
       case "morphline":
-        deriver = new MorphlineDeriver();
+        deriverClass = "com.cloudera.labs.envelope.deriver.MorphlineDeriver";
         break;
       case "pivot":
-        deriver = new PivotDeriver();
+        deriverClass = "com.cloudera.labs.envelope.deriver.PivotDeriver";
         break;
       case "exclude":
-        deriver = new ExcludeDeriver();
+        deriverClass = "com.cloudera.labs.envelope.deriver.ExcludeDeriver";
         break;
       case "dq":
-        deriver = new DataQualityDeriver();
+        deriverClass = "com.cloudera.labs.envelope.deriver.DataQualityDeriver";
         break;
       default:
-        try {
-          Class<?> clazz = Class.forName(deriverType);
-          Constructor<?> constructor = clazz.getConstructor();
-          deriver = (Deriver) constructor.newInstance();
-        }
-        catch (Exception e) {
-          throw new RuntimeException(e);
-        }
+        deriverClass = deriverType;
+    }
+
+    try {
+      Class<?> clazz = Class.forName(deriverClass);
+      Constructor<?> constructor = clazz.getConstructor();
+      deriver = (Deriver)constructor.newInstance();
+    }
+    catch (Exception e) {
+      throw new RuntimeException(e);
     }
 
     deriver.configure(config);

src/main/java/com/cloudera/labs/envelope/input/InputFactory.java → core/src/main/java/com/cloudera/labs/envelope/input/InputFactory.java

@@ -30,33 +30,36 @@ public static Input create(Config config) {
 
     String inputType = config.getString(TYPE_CONFIG_NAME);
 
+    String inputClass;
     Input input;
 
     switch (inputType) {
       case "kafka":
-        input = new KafkaInput();
+        inputClass = "com.cloudera.labs.envelope.input.KafkaInput";
         break;
       case "kudu":
-        input = new KuduInput();
+        inputClass = "com.cloudera.labs.envelope.input.KuduInput";
         break;
       case "filesystem":
-        input = new FileSystemInput();
+        inputClass = "com.cloudera.labs.envelope.input.FileSystemInput";
         break;
       case "hive":
-        input = new HiveInput();
+        inputClass = "com.cloudera.labs.envelope.input.HiveInput";
         break;
       case "jdbc":
-        input = new JdbcInput();
+        inputClass = "com.cloudera.labs.envelope.input.JdbcInput";
         break;
       default:
-        try {
-          Class<?> clazz = Class.forName(inputType);
-          Constructor<?> constructor = clazz.getConstructor();
-          input = (Input)constructor.newInstance();
-        }
-        catch (Exception e) {
-          throw new RuntimeException(e);
-        }
+        inputClass = inputType;
+    }
+
+    try {
+      Class<?> clazz = Class.forName(inputClass);
+      Constructor<?> constructor = clazz.getConstructor();
+      input = (Input)constructor.newInstance();
+    }
+    catch (Exception e) {
+      throw new RuntimeException(e);
     }
 
     input.configure(config);

src/main/java/com/cloudera/labs/envelope/input/translate/TranslatorFactory.java → core/src/main/java/com/cloudera/labs/envelope/input/translate/TranslatorFactory.java

@@ -30,29 +30,32 @@ public class TranslatorFactory {
 
     String translatorType = config.getString(TYPE_CONFIG_NAME);
 
-    Translator<?,?> translator = null;
+    String translatorClass;
+    Translator<?, ?> translator;
 
     if (translatorType.equals("kvp")) {
-      translator = new KVPTranslator();
+      translatorClass = "com.cloudera.labs.envelope.input.translate.KVPTranslator";
     }
     else if (translatorType.equals("delimited")) {
-      translator = new DelimitedTranslator();
+      translatorClass = "com.cloudera.labs.envelope.input.translate.DelimitedTranslator";
     }
     else if (translatorType.equals("avro")) {
-      translator = new AvroTranslator();
+      translatorClass = "com.cloudera.labs.envelope.input.translate.AvroTranslator";
     }
     else if (translatorType.equals("morphline")) {
-      translator = new MorphlineTranslator<>();
+      translatorClass = "com.cloudera.labs.envelope.input.translate.MorphlineTranslator";
     }
     else {
-      try {
-        Class<?> clazz = Class.forName(translatorType);
-        Constructor<?> constructor = clazz.getConstructor();
-        translator = (Translator<?, ?>)constructor.newInstance();
-      }
-      catch (Exception e) {
-        throw new RuntimeException(e);
-      }
+      translatorClass = translatorType;
+    }
+
+    try {
+      Class<?> clazz = Class.forName(translatorClass);
+      Constructor<?> constructor = clazz.getConstructor();
+      translator = (Translator<?, ?>)constructor.newInstance();
+    }
+    catch (Exception e) {
+      throw new RuntimeException(e);
     }
 
     translator.configure(config);

src/main/java/com/cloudera/labs/envelope/output/OutputFactory.java → core/src/main/java/com/cloudera/labs/envelope/output/OutputFactory.java

@@ -30,42 +30,45 @@ public static Output create(Config config) {
 
     String outputType = config.getString(TYPE_CONFIG_NAME);
 
+    String outputClass;
     Output output;
 
     switch (outputType) {
       case "kudu":
-        output = new KuduOutput();
+        outputClass = "com.cloudera.labs.envelope.output.KuduOutput";
         break;
       case "kafka":
-        output = new KafkaOutput();
+        outputClass = "com.cloudera.labs.envelope.output.KafkaOutput";
         break;
       case "log":
-        output = new LogOutput();
+        outputClass = "com.cloudera.labs.envelope.output.LogOutput";
         break;
       case "hive":
-        output = new HiveOutput();
+        outputClass = "com.cloudera.labs.envelope.output.HiveOutput";
         break;
       case "filesystem":
-        output = new FileSystemOutput();
+        outputClass = "com.cloudera.labs.envelope.output.FileSystemOutput";
         break;
       case "jdbc":
-        output = new JdbcOutput();
+        outputClass = "com.cloudera.labs.envelope.output.JdbcOutput";
         break;
       case "hbase":
-        output = new HBaseOutput();
+        outputClass = "com.cloudera.labs.envelope.output.HBaseOutput";
         break;
       case "zookeeper":
-        output = new ZooKeeperOutput();
+        outputClass = "com.cloudera.labs.envelope.output.ZooKeeperOutput";
         break;
       default:
-        try {
-          Class<?> clazz = Class.forName(outputType);
-          Constructor<?> constructor = clazz.getConstructor();
-          output = (Output)constructor.newInstance();
-        }
-        catch (Exception e) {
-          throw new RuntimeException(e);
-        }
+        outputClass = outputType;
+    }
+
+    try {
+      Class<?> clazz = Class.forName(outputClass);
+      Constructor<?> constructor = clazz.getConstructor();
+      output = (Output)constructor.newInstance();
+    }
+    catch (Exception e) {
+      throw new RuntimeException(e);
     }
 
     output.configure(config);

src/main/java/com/cloudera/labs/envelope/partition/PartitionerFactory.java → core/src/main/java/com/cloudera/labs/envelope/partition/PartitionerFactory.java

@@ -43,7 +43,8 @@ public static Partitioner create(Config config, JavaPairRDD<Row, Row> rdd) {
       throw new RuntimeException("Partitioner type not specified");
     }
 
-    Partitioner partitioner;
+    String partitionerClass = null;
+    Partitioner partitioner = null;
 
     switch (partitionerType) {
       case "hash":
@@ -55,17 +56,21 @@ public static Partitioner create(Config config, JavaPairRDD<Row, Row> rdd) {
         partitioner = new RangePartitioner<Row, Row>(rdd.getNumPartitions(), rdd.rdd(), true, rowOrdering, rowClassTag);
         break;
       case "uuid":
-        partitioner = new UUIDPartitioner();
+        partitionerClass = "com.cloudera.labs.envelope.partition.UUIDPartitioner";
         break;
       default:
-        try {
-          Class<?> clazz = Class.forName(partitionerType);
-          Constructor<?> constructor = clazz.getConstructor();
-          partitioner = (Partitioner)constructor.newInstance();
-        }
-        catch (Exception e) {
-          throw new RuntimeException(e);
-        }
+        partitionerClass = partitionerType;
+    }
+
+    if (partitioner == null) {
+      try {
+        Class<?> clazz = Class.forName(partitionerClass);
+        Constructor<?> constructor = clazz.getConstructor();
+        partitioner = (Partitioner)constructor.newInstance();
+      }
+      catch (Exception e) {
+        throw new RuntimeException(e);
+      }
     }
 
     if (partitioner instanceof ConfigurablePartitioner) {