Merge pull request BD2KGenomics#292 from akmorrow13/spark_2.4.3

Update Mango to Spark 2.4.3

mango/Makefile

@@ -7,7 +7,7 @@ build_tool = runtime-container.DONE
 build_number ?= none
 git_commit ?= $(shell git log --pretty=oneline -n 1 -- ../mango | cut -f1 -d " ")
 name = quay.io/ucsc_cgl/mango
-tag = 0.0.1--${git_commit}
+tag = 0.0.3--${git_commit}
 
 
 # Steps

mango/build/Dockerfile

@@ -13,10 +13,18 @@ WORKDIR /home
 RUN git clone https://github.com/bigdatagenomics/mango.git
 ENV MAVEN_OPTS "-Xmx2g"
 
+# install curl to get nodejs script
+RUN apt-get update && apt-get install -y curl
+
+# get nodejs v6.X
+RUN curl -sL https://deb.nodesource.com/setup_8.x | bash -
+
+RUN apt-get install -y nodejs
+
 # build mango
 WORKDIR /home/mango
-
-RUN /opt/apache-maven-3.3.9/bin/mvn package -DskipTests
+RUN git checkout c85b5d2178dbf7ec84cee20c56c57493524d510e # 0.0.3-SNAPSHOT
+RUN /opt/apache-maven-3.3.9/bin/mvn clean package -DskipTests
 
 # remove git libraries to avoid permission errors when copying
 RUN rm -rf /home/mango/.git

mango/runtime/Dockerfile

@@ -6,13 +6,16 @@ MAINTAINER Alyssa Morrow, [email protected]
 RUN mkdir /opt/cgl-docker-lib
 COPY mango /opt/cgl-docker-lib/mango
 
+WORKDIR /opt/cgl-docker-lib/mango
+
 # copy spark
 COPY apache-spark /opt/cgl-docker-lib/apache-spark
 
 ENV SPARK_HOME /opt/cgl-docker-lib/apache-spark
 
 # put mango jar on the pyspark path for packaging
 ENV ASSEMBLY_DIR /opt/cgl-docker-lib/mango/mango-assembly/target
+
 ENV ASSEMBLY_JAR "$(ls -1 "$ASSEMBLY_DIR" | grep "^mango-assembly[0-9A-Za-z\_\.-]*\.jar$" | grep -v javadoc | grep -v sources || true)"
 ENV PYSPARK_SUBMIT_ARGS "--jars ${ASSEMBLY_DIR}/${ASSEMBLY_JAR} --driver-class-path ${ASSEMBLY_DIR}/${ASSEMBLY_JAR} pyspark-shell"
 
@@ -22,17 +25,23 @@ ENV PYTHONPATH ${SPARK_HOME}/python:${SPARK_HOME}/python/lib/py4j-0.10.4-src.zip
 #environment variables PYSPARK_PYTHON and PYSPARK_DRIVER_PYTHON
 ENV PYSPARK_PYTHON /usr/bin/python3
 
-# Install make and pip/python dependencies
 RUN apt-get update && apt-get install -y \
-  git \
-  make \
-  python3.5 \
-  python3-pip \
-  npm \
-  nodejs \
-  python-tk
+   git \
+   make \
+   python3.5 \
+   python3-pip \
+   python-tk \
+   curl
+
+# get nodejs v8.X
+RUN apt-cache policy nodejs
+RUN curl -sL https://deb.nodesource.com/setup_8.x | bash -
+RUN apt-cache policy nodejs
 
 
+# Install make and pip/python dependencies
+RUN apt-get install -y nodejs
+
 # alias python to python3.6
 RUN alias pip=pip3
 RUN alias python=python3
@@ -43,13 +52,9 @@ RUN echo '#!/bin/bash\npython3 "$@"' > /usr/bin/python && \
 RUN echo '#!/bin/bash\npip3 "$@"' > /usr/bin/pip && \
     chmod +x /usr/bin/pip
 
-RUN pip
-RUN ln -s /usr/bin/nodejs /usr/bin/node
-
 # set permissions for running npm. Required for mango-viz
 RUN npm config set bdgenomics.mango.pileup:unsafe-perm
 
-
 # prepare mango-viz and mango-python
 WORKDIR /opt/cgl-docker-lib/mango/mango-python
 RUN make prepare && make develop

mango/test.py

@@ -10,7 +10,7 @@ class TestMango(unittest.TestCase):
     def test_docker_call_browser(self):
         out, err = check_docker_output(tool='quay.io/ucsc_cgl/mango')
         self.assertTrue('Using spark-submit=' in out)
-        self.assertTrue('Argument "reference" is required' in out)
+        self.assertTrue('Argument "genome" is required' in out)
 
     def test_docker_call_notebook(self):
         out, err = check_docker_output(tool='--entrypoint=/opt/cgl-docker-lib/mango/bin/mango-notebook quay.io/ucsc_cgl/mango')

spark-and-maven/build/download.sh

@@ -7,7 +7,7 @@ mirror=$(python -c "from urllib2 import urlopen; import json; print json.load( u
 
 # pull down spark
 mkdir /opt/apache-spark
-curl ${mirror}spark/spark-2.3.2/spark-2.3.2-bin-hadoop2.7.tgz \
+curl ${mirror}spark/spark-2.4.3/spark-2.4.3-bin-hadoop2.7.tgz \
   | tar --strip-components=1 -xzC /opt/apache-spark
 
 # we rely on apache maven > 3.1.1 to build ADAM, so we can't use the