Merge pull request BD2KGenomics#285 from akmorrow13/master

updated Mango for Spark 2.3.2
amanpruthi · Nov 30, 2018 · 43525f5 · 43525f5
2 parents dc2dc31 + e18ed93
commit 43525f5
Show file tree

Hide file tree

Showing 5 changed files with 16 additions and 11 deletions.
diff --git a/avocado/build/Dockerfile b/avocado/build/Dockerfile
@@ -9,7 +9,5 @@ RUN git clone https://github.com/bigdatagenomics/avocado.git
 
 # build avocado
 WORKDIR /home/avocado
-RUN ./scripts/move_to_spark_2.sh
-RUN ./scripts/move_to_scala_2.11.sh
 
 RUN /opt/apache-maven-3.3.9/bin/mvn package -DskipTests
diff --git a/cannoli/build/Dockerfile b/cannoli/build/Dockerfile
@@ -9,8 +9,6 @@ RUN git clone https://github.com/bigdatagenomics/cannoli.git
 
 # build cannoli
 WORKDIR /home/cannoli
-RUN git checkout 122ead2219b987bb60e5f108a2d261e056abc86e
-RUN ./scripts/move_to_spark_2.sh
-RUN ./scripts/move_to_scala_2.11.sh
+RUN git checkout 4395e5b2e6040cbe5a24398d87ae49535fa62a23
 
 RUN /opt/apache-maven-3.3.9/bin/mvn package -DskipTests
diff --git a/mango/runtime/Dockerfile b/mango/runtime/Dockerfile
@@ -20,24 +20,33 @@ ENV PYSPARK_SUBMIT_ARGS "--jars ${ASSEMBLY_DIR}/${ASSEMBLY_JAR} --driver-class-p
 ENV PYTHONPATH ${SPARK_HOME}/python:${SPARK_HOME}/python/lib/py4j-0.10.4-src.zip
 
 #environment variables PYSPARK_PYTHON and PYSPARK_DRIVER_PYTHON
-ENV PYSPARK_PYTHON /usr/bin/python
-ENV  PYSPARK_DRIVER_PYTHON /usr/bin/python
-
+ENV PYSPARK_PYTHON /usr/bin/python3
 
 # Install make and pip/python dependencies
 RUN apt-get update && apt-get install -y \
   git \
   make \
-  python-pip \
+  python3-pip \
   npm \
   nodejs \
   python-tk
 
+# alias python to python3.6
+RUN alias pip=pip3
+RUN alias python=python3
+RUN echo '#!/bin/bash\npython3 "$@"' > /usr/bin/python && \
+    chmod +x /usr/bin/python
+
+RUN echo '#!/bin/bash\npip3 "$@"' > /usr/bin/pip && \
+    chmod +x /usr/bin/pip
+
+RUN pip
 RUN ln -s /usr/bin/nodejs /usr/bin/node
 
 # set permissions for running npm. Required for mango-viz
 RUN npm config set bdgenomics.mango.pileup:unsafe-perm
 
+
 # prepare mango-viz and mango-python
 WORKDIR /opt/cgl-docker-lib/mango/mango-python
 RUN make prepare && make develop

diff --git a/mango/test.py b/mango/test.py
@@ -9,7 +9,7 @@ class TestMango(unittest.TestCase):
 
     def test_docker_call_browser(self):
         out, err = check_docker_output(tool='quay.io/ucsc_cgl/mango')
-        self.assertTrue('Using SPARK_SUBMIT=' in out)
+        self.assertTrue('Using spark-submit=' in out)
         self.assertTrue('Argument "reference" is required' in out)
 
     def test_docker_call_notebook(self):

diff --git a/spark-and-maven/build/download.sh b/spark-and-maven/build/download.sh
@@ -7,7 +7,7 @@ mirror=$(python -c "from urllib2 import urlopen; import json; print json.load( u
 
 # pull down spark
 mkdir /opt/apache-spark
-curl ${mirror}spark/spark-2.1.2/spark-2.1.2-bin-hadoop2.6.tgz \
+curl ${mirror}spark/spark-2.3.2/spark-2.3.2-bin-hadoop2.7.tgz \
   | tar --strip-components=1 -xzC /opt/apache-spark
 
 # we rely on apache maven > 3.1.1 to build ADAM, so we can't use the