Skip to content

Commit

Permalink
Hadoop Scalding Runner wrapper
Browse files Browse the repository at this point in the history
  • Loading branch information
Renien committed Mar 13, 2017
1 parent a2d5ca1 commit 41b0cc1
Showing 1 changed file with 35 additions and 0 deletions.
35 changes: 35 additions & 0 deletions Common/src/main/scala/com/etl/utils/HadoopRunner.scala
Original file line number Diff line number Diff line change
@@ -0,0 +1,35 @@
package com.etl.utils

import java.util.Properties

import com.twitter.scalding.{RichXHandler, Tool}
import org.apache.hadoop

/**
* Created by renienj on 3/12/17.
*/
class HadoopRunner(job: String, prop: Properties) {

def run() {
val args = prop.get("main.args").asInstanceOf[String]
require(args != null, "unable to get main.args")
try {
hadoop.util.ToolRunner.run(new hadoop.conf.Configuration, new Tool, args.split(" "))
} catch {
case t: Throwable => {
//create the exception URL link in GitHub wiki
val gitHubLink = RichXHandler.createXUrl(t)
val extraInfo = (if (RichXHandler().handlers.exists(h => h(t))) {
RichXHandler.mapping(t.getClass) + "\n"
}
else {
""
}) +
"If you know what exactly caused this error, please consider contributing to GitHub via following link.\n" + gitHubLink

//re-throw the exception with extra info
throw new Throwable(extraInfo, t)
}
}
}
}

0 comments on commit 41b0cc1

Please sign in to comment.