From a779840fac6fd9d17eff65ec10fd935796a87a3c Mon Sep 17 00:00:00 2001 From: Alyssa Morrow Date: Mon, 1 Jul 2019 16:49:28 -0700 Subject: [PATCH] mango docker now runs with nodejs 8.X and Spark 2.4.3 --- mango/Makefile | 2 +- mango/build/Dockerfile | 19 ++++++++----------- mango/runtime/Dockerfile | 27 ++++++++++++++------------- mango/test.py | 2 +- 4 files changed, 24 insertions(+), 26 deletions(-) diff --git a/mango/Makefile b/mango/Makefile index 07a06b7..4811f4e 100644 --- a/mango/Makefile +++ b/mango/Makefile @@ -7,7 +7,7 @@ build_tool = runtime-container.DONE build_number ?= none git_commit ?= $(shell git log --pretty=oneline -n 1 -- ../mango | cut -f1 -d " ") name = quay.io/ucsc_cgl/mango -tag = 0.0.1--${git_commit} +tag = 0.0.3--${git_commit} # Steps diff --git a/mango/build/Dockerfile b/mango/build/Dockerfile index f5e35b1..2125f79 100644 --- a/mango/build/Dockerfile +++ b/mango/build/Dockerfile @@ -13,21 +13,18 @@ WORKDIR /home RUN git clone https://github.com/bigdatagenomics/mango.git ENV MAVEN_OPTS "-Xmx2g" -RUN apt-get update && apt-get install -y \ - unzip \ - npm \ - nodejs +# install curl to get nodejs script +RUN apt-get update && apt-get install -y curl + +# get nodejs v6.X +RUN curl -sL https://deb.nodesource.com/setup_8.x | bash - + +RUN apt-get install -y nodejs # build mango WORKDIR /home/mango +RUN git checkout c85b5d2178dbf7ec84cee20c56c57493524d510e # 0.0.3-SNAPSHOT RUN /opt/apache-maven-3.3.9/bin/mvn clean package -DskipTests -RUN ls /home/mango/mango-cli/src/main/webapp/resources - -# where is node_modules? -RUN mkdir /home/mango/mango-assembly/target/test -WORKDIR /home/mango/mango-assembly/target -RUN unzip mango-assembly-0.0.3-SNAPSHOT.jar -d /home/mango/mango-assembly/target/test -RUN ls test/resources/node_modules/pileup # remove git libraries to avoid permission errors when copying RUN rm -rf /home/mango/.git diff --git a/mango/runtime/Dockerfile b/mango/runtime/Dockerfile index 59c34b9..5694e48 100644 --- a/mango/runtime/Dockerfile +++ b/mango/runtime/Dockerfile @@ -15,7 +15,6 @@ ENV SPARK_HOME /opt/cgl-docker-lib/apache-spark # put mango jar on the pyspark path for packaging ENV ASSEMBLY_DIR /opt/cgl-docker-lib/mango/mango-assembly/target -RUN ls /opt/cgl-docker-lib/mango/mango-cli/src/main/webapp/resources ENV ASSEMBLY_JAR "$(ls -1 "$ASSEMBLY_DIR" | grep "^mango-assembly[0-9A-Za-z\_\.-]*\.jar$" | grep -v javadoc | grep -v sources || true)" ENV PYSPARK_SUBMIT_ARGS "--jars ${ASSEMBLY_DIR}/${ASSEMBLY_JAR} --driver-class-path ${ASSEMBLY_DIR}/${ASSEMBLY_JAR} pyspark-shell" @@ -26,16 +25,22 @@ ENV PYTHONPATH ${SPARK_HOME}/python:${SPARK_HOME}/python/lib/py4j-0.10.4-src.zip #environment variables PYSPARK_PYTHON and PYSPARK_DRIVER_PYTHON ENV PYSPARK_PYTHON /usr/bin/python3 -# Install make and pip/python dependencies RUN apt-get update && apt-get install -y \ - git \ - make \ - python3.5 \ - python3-pip \ - npm \ - nodejs \ - python-tk + git \ + make \ + python3.5 \ + python3-pip \ + python-tk \ + curl + +# get nodejs v8.X +RUN apt-cache policy nodejs +RUN curl -sL https://deb.nodesource.com/setup_8.x | bash - +RUN apt-cache policy nodejs + +# Install make and pip/python dependencies +RUN apt-get install -y nodejs # alias python to python3.6 RUN alias pip=pip3 @@ -47,13 +52,9 @@ RUN echo '#!/bin/bash\npython3 "$@"' > /usr/bin/python && \ RUN echo '#!/bin/bash\npip3 "$@"' > /usr/bin/pip && \ chmod +x /usr/bin/pip -RUN pip -RUN ln -s /usr/bin/nodejs /usr/bin/node - # set permissions for running npm. Required for mango-viz RUN npm config set bdgenomics.mango.pileup:unsafe-perm - # prepare mango-viz and mango-python WORKDIR /opt/cgl-docker-lib/mango/mango-python RUN make prepare && make develop diff --git a/mango/test.py b/mango/test.py index eeadaea..ef7972d 100644 --- a/mango/test.py +++ b/mango/test.py @@ -10,7 +10,7 @@ class TestMango(unittest.TestCase): def test_docker_call_browser(self): out, err = check_docker_output(tool='quay.io/ucsc_cgl/mango') self.assertTrue('Using spark-submit=' in out) - self.assertTrue('Argument "reference" is required' in out) + self.assertTrue('Argument "genome" is required' in out) def test_docker_call_notebook(self): out, err = check_docker_output(tool='--entrypoint=/opt/cgl-docker-lib/mango/bin/mango-notebook quay.io/ucsc_cgl/mango')