build.gradle

plugins {
  id 'org.hidetake.ssh' version '1.5.0'
}

// load some common helper methods
apply from: "${projectDir}/../../shared/common-helpers.gradle"

// get the cluster connection details
Properties props = new Properties()
props.load(new FileInputStream("$projectDir/../../connection.properties"))


task PythonWordCount << {

    def tmpDir = "test-${new Date().getTime()}"

    def tmpHdfsDir = "/user/${props.username}/${tmpDir}"
    
    // ssh plugin documentation: https://gradle-ssh-plugin.github.io/docs/
    
    ssh.run {
        // remotes.bicluster is defined in shared/common-helpers.gradle
        session(remotes.bicluster) {

            try {
                // initialise kerberos
                execute "kinit -k -t ${props.username}.keytab ${props.username}@IBM.COM"
            } 
            catch (Exception e) {
                println "problem running kinit - maybe this is a Basic cluster?"
            }

            // create temp local dir for holding LICENSE file before uploading it to hdfs
            execute "mkdir ${tmpDir}"

            // upload spark script and text file to process
            put from: "${projectDir}/wordcount.py", into: "${tmpDir}/wordcount.py"
            put from: "${projectDir}/LICENSE", into: "${tmpDir}/LICENSE"

            // create temp hdfs folder for holding LICENSE file
            execute "hadoop fs -mkdir ${tmpHdfsDir}"
            
            // put LICENSE into hdfs
            execute "hadoop fs -put ${tmpDir}/LICENSE ${tmpHdfsDir}/LICENSE"

            // execute spark workcount job against the LICENSE file in hdfs
            execute "pyspark ${tmpDir}/wordcount.py ${tmpHdfsDir}/LICENSE"

            // remove temporary hdfs dir
            execute "hadoop fs -rm -r ${tmpHdfsDir}"

            // remove temporary local dir
            execute "rm -rf ${tmpDir}"
        }
    }
}


task Example {
	dependsOn PythonWordCount
}