Support for embedding Pig scripts in Java test files

Uses a neat trick presented by Adrian Walker in this blog post: http://www.adrianwalker.org/2011/12/java-multiline-string.html

With this the pig scripts can be embedded directly in the Java files as javadoc comments on class fields.  Tests are much easier to work with this way.

I removed all pig test files for the util tests to demonstrate.

Also added an ant target "eclipse" which generates some eclipse files.  It figures out what the paths should be based on the environment.  This fixes an issue where paths would be incorrect when working in another repo.

Author: Matt Hayes

.classpath .classpath.template

@@ -5,7 +5,13 @@
 	<classpathentry kind="src" path="test/pig"/>
 	<classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER"/>
 	<classpathentry kind="con" path="org.testng.TESTNG_CONTAINER"/>
-	<classpathentry kind="con" path="org.apache.ivyde.eclipse.cpcontainer.IVYDE_CONTAINER/?project=datafu&amp;ivyXmlPath=ivy.xml&amp;confs=*"/>
-	<classpathentry kind="lib" path="otherlibs/pigunit-0.9.jar"/>
-	<classpathentry kind="output" path="dist/testclasses"/>
+	<classpathentry kind="con" path="org.apache.ivyde.eclipse.cpcontainer.IVYDE_CONTAINER/?project=@PROJECT@&amp;ivyXmlPath=ivy.xml&amp;confs=*"/>
+	<classpathentry kind="src" path=".apt_generated">
+		<attributes>
+			<attribute name="optional" value="true"/>
+		</attributes>
+	</classpathentry>
+	<classpathentry kind="lib" path="annotation-plugin.jar"/>
+	<classpathentry kind="lib" path="staticlibs/pigunit-0.9.jar"/>
+	<classpathentry kind="output" path="build/testclasses"/>
 </classpath>

.factorypath.template

@@ -0,0 +1,3 @@
+<factorypath>
+    <factorypathentry kind="WKSPJAR" id="@HOME@/annotation-plugin.jar" enabled="true" runInBatchMode="false"/>
+</factorypath>

.gitignore

@@ -1,4 +1,8 @@
 dist/
+build/
+annotation-plugin.jar
+.factorypath
+.classpath
 docs/
 input
 lib/

.settings/org.eclipse.jdt.apt.core.prefs

@@ -0,0 +1,4 @@
+eclipse.preferences.version=1
+org.eclipse.jdt.apt.aptEnabled=true
+org.eclipse.jdt.apt.genSrcDir=.apt_generated
+org.eclipse.jdt.apt.reconcileEnabled=true

.settings/org.eclipse.jdt.core.prefs

@@ -0,0 +1,12 @@
+eclipse.preferences.version=1
+org.eclipse.jdt.core.compiler.codegen.inlineJsrBytecode=enabled
+org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.6
+org.eclipse.jdt.core.compiler.codegen.unusedLocal=preserve
+org.eclipse.jdt.core.compiler.compliance=1.6
+org.eclipse.jdt.core.compiler.debug.lineNumber=generate
+org.eclipse.jdt.core.compiler.debug.localVariable=generate
+org.eclipse.jdt.core.compiler.debug.sourceFile=generate
+org.eclipse.jdt.core.compiler.problem.assertIdentifier=error
+org.eclipse.jdt.core.compiler.problem.enumIdentifier=error
+org.eclipse.jdt.core.compiler.processAnnotations=enabled
+org.eclipse.jdt.core.compiler.source=1.6

README.md

@@ -151,6 +151,12 @@ Or [download](https://github.com/linkedin/datafu/archive/master.zip) the code.
 
 Here are some common tasks when working with the source code.
 
+### Eclipse
+
+To generate eclipse files:
+
+    ant eclipse
+
 ### Build the JAR
 
     ant jar

build.xml

@@ -20,13 +20,16 @@
   <!-- directories -->
   <property name="src.dir" value="${basedir}/src" />
   <property name="java.dir" value="${src.dir}/java" />
+  <property name="plugin.java.dir" value="${basedir}/plugin/java" />
   <property name="test.dir" value="${basedir}/test" />
   <property name="unittestsrc.dir" value="${test.dir}/unit" />
   <property name="pigtestsrc.dir" value="${test.dir}/pig" />
   <property name="dist.dir" value="${basedir}/dist" />
   <property name="tools.dir" value="${basedir}/tools" />
   <property name="lib.dir" value="${basedir}/lib" />
+  <property name="static.lib.dir" value="${basedir}/staticlibs" />
   <property name="classes.dir" value="${dist.dir}/classes" />
+  <property name="plugin.classes.dir" value="${dist.dir}/pluginclasses" />
   <property name="report.dir" value="${basedir}/report" />
   <property name="report.unit.dir" value="${report.dir}/unit" />
   <property name="report.pig.dir" value="${report.dir}/pig" />
@@ -59,12 +62,37 @@
 	<property name="maven.jar" location="${ivy.jar.dir}/maven-ant-tasks-${maven.jar.version}.jar"/>
 
   <loadproperties srcfile="${ivy.jar.dir}/libraries.properties"/>
+
+  <basename property="basedirname" file="${basedir}"/>
 
   <target name="all" depends="clean, jar, sources-jar, javadoc-jar, coverage" description="Build all artifacts." />
 
   <target name="init" depends="ivy-resolve">
     <mkdir dir="${dist.dir}" />
   </target>
+
+  <target name="eclipse" depends="jar,jar-plugin">
+    <delete file="${basedir}/.factorypath"/>
+    <copy file="${basedir}/.factorypath.template"
+             tofile="${basedir}/.factorypath">
+        <filterchain>
+           <replacetokens>
+                <token key="HOME" value="/${basedirname}"/>
+            </replacetokens>
+        </filterchain>
+    </copy>
+    <echo message="Created .factorypath"/>
+    <delete file="${basedir}/.classpath"/>
+    <copy file="${basedir}/.classpath.template"
+             tofile="${basedir}/.classpath">
+        <filterchain>
+           <replacetokens>
+                <token key="PROJECT" value="${basedirname}"/>
+            </replacetokens>
+        </filterchain>
+    </copy>
+    <echo message="Created .classpath"/>
+  </target>
 
   <target name="maven-ant-tasks-jar-download" description="Download maven tasks jar">
     <mkdir dir="${ivy.jar.dir}"/>
@@ -119,11 +147,17 @@
 
   <taskdef classpathref="cobertura.classpath" resource="tasks.properties" />
 
+  <path id="plugin-classpath">
+    <fileset dir="${static.lib.dir}">
+      <include name="*.jar" />
+    </fileset>
+  </path>
+
   <path id="base-test-classpath">    
     <fileset dir="${lib.dir}">
       <include name="*.jar" />
     </fileset>
-    <fileset dir="${basedir}/otherlibs">
+    <fileset dir="${static.lib.dir}">
       <include name="*.jar" />
     </fileset>
     <pathelement path="${classes.dir}" />
@@ -137,6 +171,7 @@
     <fileset dir="${dist.dir}">
       <include name="*.jar" />
     </fileset>
+    <pathelement path="${plugin.classes.dir}" />
   </path>
 
   <path id="instrumented-test-classpath">
@@ -157,6 +192,7 @@
   </macrodef>
 
   <target name="clean" description="Delete generated files.">
+    <delete dir="${basedir}/build" />
     <delete dir="${lib.dir}" />
     <delete dir="${dist.dir}" />
     <delete dir="${docs.dir}" />
@@ -175,6 +211,24 @@
       <classpath refid="main-classpath" />
     </javac>
   </target>
+
+  <target name="build-plugins" depends="init" description="Compile main source tree java files">
+    <replace-dir dir="${plugin.classes.dir}" />
+    <javac destdir="${plugin.classes.dir}" target="1.5" debug="true" deprecation="false" failonerror="true" includeantruntime="false">
+      <src path="${plugin.java.dir}" />
+      <classpath refid="plugin-classpath" />
+    </javac>
+  </target>
+
+  <target name="jar-plugin" depends="build-plugins" description="Builds the plugin JAR">
+    <delete file="${basedir}/annotation-plugin.jar" />
+    <jar destfile="${basedir}/annotation-plugin.jar">
+      <fileset dir="${plugin.classes.dir}">
+        <include name="**/*.*" />
+      </fileset>
+      <service type="javax.annotation.processing.Processor" provider="org.adrianwalker.multilinestring.MultilineProcessor"/>
+    </jar>
+  </target>
 
   <target name="build-unit-tests" depends="build" description="Compile unit test classes">
     <replace-dir dir="${unittestclasses.dir}" />
@@ -184,7 +238,7 @@
     </javac>
   </target>
 
-  <target name="build-pig-tests" depends="build, build-unit-tests" description="Compile pig test classes">
+  <target name="build-pig-tests" depends="build,build-unit-tests,build-plugins" description="Compile pig test classes">
     <replace-dir dir="${pigtestclasses.dir}" />
     <copy todir="${pigtestclasses.dir}">
       <fileset dir="${pigtestsrc.dir}">
@@ -193,6 +247,7 @@
     </copy>
     <javac destdir="${pigtestclasses.dir}" target="1.5" debug="true" deprecation="false" failonerror="true" includeantruntime="false">
       <src path="${pigtestsrc.dir}" />
+      <compilerarg line="-processor org.adrianwalker.multilinestring.MultilineProcessor"/>
       <classpath refid="test-classpath" />
     </javac>
   </target>

eclipselibs/org.eclipse.jdt.compiler.apt_1.0.500.v20120522-1651.jar

@@ -0,0 +1,51 @@
+package org.adrianwalker.multilinestring;
+
+import java.util.Set;
+
+import javax.annotation.processing.AbstractProcessor;
+import javax.annotation.processing.ProcessingEnvironment;
+import javax.annotation.processing.RoundEnvironment;
+import javax.annotation.processing.SupportedAnnotationTypes;
+import javax.annotation.processing.SupportedSourceVersion;
+import javax.lang.model.SourceVersion;
+import javax.lang.model.element.Element;
+import javax.lang.model.element.TypeElement;
+import javax.lang.model.util.Elements;
+
+import org.eclipse.jdt.internal.compiler.apt.model.VariableElementImpl;
+import org.eclipse.jdt.internal.compiler.ast.FieldDeclaration;
+import org.eclipse.jdt.internal.compiler.ast.StringLiteral;
+import org.eclipse.jdt.internal.compiler.lookup.FieldBinding;
+
+import java.lang.reflect.Constructor;
+
+@SupportedAnnotationTypes({"org.adrianwalker.multilinestring.Multiline"})
+@SupportedSourceVersion(SourceVersion.RELEASE_6)
+public final class EcjMultilineProcessor extends AbstractProcessor {
+
+  private Elements elementUtils;
+  
+  @Override
+  public void init(final ProcessingEnvironment procEnv) {
+    super.init(procEnv);
+    this.elementUtils = procEnv.getElementUtils();
+  }
+
+  @Override
+  public boolean process(final Set<? extends TypeElement> annotations, final RoundEnvironment roundEnv) {
+    Set<? extends Element> fields = roundEnv.getElementsAnnotatedWith(Multiline.class);
+
+    for (Element field : fields) {
+      String docComment = elementUtils.getDocComment(field);
+
+      if (null != docComment) {
+        VariableElementImpl fieldElem = (VariableElementImpl) field;
+        FieldBinding biding = (FieldBinding) fieldElem._binding;
+        FieldDeclaration decl = biding.sourceField();
+        StringLiteral string = new StringLiteral(docComment.toCharArray(), decl.sourceStart, decl.sourceEnd, decl.sourceStart);
+        decl.initialization = string;
+      }
+    }
+    return true;
+  }
+}

eclipselibs/org.eclipse.jdt.core_3.8.3.v20130121-145325.jar

@@ -0,0 +1,46 @@
+package org.adrianwalker.multilinestring;
+
+import java.util.Set;
+
+import javax.annotation.processing.AbstractProcessor;
+import javax.annotation.processing.ProcessingEnvironment;
+import javax.annotation.processing.RoundEnvironment;
+import javax.annotation.processing.SupportedAnnotationTypes;
+import javax.annotation.processing.SupportedSourceVersion;
+import javax.lang.model.SourceVersion;
+import javax.lang.model.element.Element;
+import javax.lang.model.element.TypeElement;
+
+import com.sun.tools.javac.model.JavacElements;
+import com.sun.tools.javac.processing.JavacProcessingEnvironment;
+import com.sun.tools.javac.tree.JCTree.JCVariableDecl;
+import com.sun.tools.javac.tree.TreeMaker;
+
+@SupportedAnnotationTypes({"org.adrianwalker.multilinestring.Multiline"})
+@SupportedSourceVersion(SourceVersion.RELEASE_6)
+public final class JavacMultilineProcessor extends AbstractProcessor {
+
+  private JavacElements elementUtils;
+  private TreeMaker maker;
+  
+  @Override
+  public void init(final ProcessingEnvironment procEnv) {
+    super.init(procEnv);
+    JavacProcessingEnvironment javacProcessingEnv = (JavacProcessingEnvironment) procEnv;
+    this.elementUtils = javacProcessingEnv.getElementUtils();
+    this.maker = TreeMaker.instance(javacProcessingEnv.getContext());
+  }
+
+  @Override
+  public boolean process(final Set<? extends TypeElement> annotations, final RoundEnvironment roundEnv) {
+    Set<? extends Element> fields = roundEnv.getElementsAnnotatedWith(Multiline.class);
+    for (Element field : fields) {
+      String docComment = elementUtils.getDocComment(field);
+      if (null != docComment) {
+        JCVariableDecl fieldNode = (JCVariableDecl) elementUtils.getTree(field);
+        fieldNode.init = maker.Literal(docComment);
+      }
+    }
+    return true;
+  }
+}

plugin/java/org/adrianwalker/multilinestring/EcjMultilineProcessor.java

@@ -0,0 +1,11 @@
+package org.adrianwalker.multilinestring;
+
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+
+@Target({ElementType.FIELD,ElementType.LOCAL_VARIABLE})
+@Retention(RetentionPolicy.SOURCE)
+public @interface Multiline {
+}

plugin/java/org/adrianwalker/multilinestring/JavacMultilineProcessor.java

@@ -0,0 +1,38 @@
+package org.adrianwalker.multilinestring;
+
+import java.util.Set;
+
+import javax.annotation.processing.AbstractProcessor;
+import javax.annotation.processing.ProcessingEnvironment;
+import javax.annotation.processing.Processor;
+import javax.annotation.processing.RoundEnvironment;
+import javax.annotation.processing.SupportedAnnotationTypes;
+import javax.annotation.processing.SupportedSourceVersion;
+import javax.lang.model.SourceVersion;
+import javax.lang.model.element.TypeElement;
+
+@SupportedAnnotationTypes({"org.adrianwalker.multilinestring.Multiline"})
+@SupportedSourceVersion(SourceVersion.RELEASE_6)
+public final class MultilineProcessor extends AbstractProcessor {
+  private Processor delegator = null;
+  
+  @Override
+  public void init(final ProcessingEnvironment procEnv) {
+    super.init(procEnv);
+    String envClassName = procEnv.getClass().getName();
+    if (envClassName.contains("com.sun.tools")) {
+      delegator = new JavacMultilineProcessor();
+    } else {
+      delegator = new EcjMultilineProcessor();
+    }
+    delegator.init(procEnv);
+  }
+
+  @Override
+  public boolean process(final Set<? extends TypeElement> annotations, final RoundEnvironment roundEnv) {
+    if (delegator == null ) {
+      return true;
+    }
+    return delegator.process(annotations, roundEnv);
+  }
+}

plugin/java/org/adrianwalker/multilinestring/Multiline.java

@@ -56,7 +56,12 @@ protected List<String> getDefaultArgsAsList()
     return argsList;
   }
 
-  protected PigTest createPigTest(String scriptPath, String... args) throws IOException
+  protected PigTest createPigTestFromString(String str, String... args) throws IOException
+  {
+    return createPigTest(str.split("\n"),args);
+  }
+  
+  protected PigTest createPigTest(String[] lines, String... args) throws IOException
   {
     // append args to list of default args
     List<String> theArgs = getDefaultArgsAsList();
@@ -65,8 +70,6 @@ protected PigTest createPigTest(String scriptPath, String... args) throws IOExce
       theArgs.add(arg);
     }
 
-    String[] lines = getLinesFromFile(scriptPath);
-    
     for (String arg : theArgs)
     {
       String[] parts = arg.split("=",2);
@@ -82,9 +85,9 @@ protected PigTest createPigTest(String scriptPath, String... args) throws IOExce
     return new PigTest(lines);
   }
 
-  protected PigTest createPigTest(String scriptPath) throws IOException
+  protected PigTest createPigTest(String scriptPath, String... args) throws IOException
   {
-    return createPigTest(scriptPath, getDefaultArgs());
+    return createPigTest(getLinesFromFile(scriptPath), args);
   }
 
   protected String getJarPath()