devitocodes · georgebisbas · Feb 12, 2020 · Feb 12, 2020 · Feb 12, 2020 · Feb 12, 2020
diff --git a/docker/Dockerfile → recipes/devito-gcc-mpich/docker/Dockerfile b/docker/Dockerfile → recipes/devito-gcc-mpich/docker/Dockerfile
@@ -1,4 +1,5 @@
 FROM ubuntu:18.04
+RUN rm /bin/sh && ln -s /bin/bash /bin/sh
 
 # set up base and ssh keys
 COPY ssh_config /root/.ssh/config

diff --git a/docker/ssh_config → recipes/devito-gcc-mpich/docker/ssh_config b/docker/ssh_config → recipes/devito-gcc-mpich/docker/ssh_config
diff --git a/shipyard-config/config.yaml → ...ito-gcc-mpich/shipyard-config/config.yaml b/shipyard-config/config.yaml → ...ito-gcc-mpich/shipyard-config/config.yaml
diff --git a/shipyard-config/jobs.yaml → ...evito-gcc-mpich/shipyard-config/jobs.yaml b/shipyard-config/jobs.yaml → ...evito-gcc-mpich/shipyard-config/jobs.yaml
@@ -19,7 +19,7 @@ job_specifications:
       num_instances: pool_current_dedicated
       mpi:
         runtime: mpich
-        processes_per_node: 2
+        processes_per_node: 1
         options:
           - --bind-to socket
       pre_execution_command: lscpu;

diff --git a/recipes/devito-gcc-mpich/shipyard-config/pool.yaml b/recipes/devito-gcc-mpich/shipyard-config/pool.yaml
@@ -0,0 +1,16 @@
+pool_specification:
+  id: devito-gcc-mpich8
+  vm_configuration:
+    platform_image:
+      offer: UbuntuServer
+      publisher: Canonical
+      sku: 18.04-LTS
+      native: true
+  vm_count:
+    dedicated: 4
+    low_priority: 0
+  resize_timeout: 00:40:00
+  vm_size: Standard_H8
+  inter_node_communication_enabled: true
+  ssh:
+    username: shipyard
diff --git a/recipes/devito-icc-intelmpi-drivers/docker/Dockerfile b/recipes/devito-icc-intelmpi-drivers/docker/Dockerfile
@@ -0,0 +1,113 @@
+FROM centos:7.6.1810
+RUN rm /bin/sh && ln -s /bin/bash /bin/sh
+
+RUN yum install -y gcc gcc-c++ make vim htop
+
+# ICC
+COPY conf_mpi_test.cfg /tmp/icc-config.cfg
+ADD ./parallel_studio_xe_2020_cluster_edition_online /tmp/intel
+
+
+RUN cd /tmp/intel && \
+  ./install.sh --silent=/tmp/icc-config.cfg && \
+  cd ../../ && \
+  rm -rf /tmp/intel && \
+  rm /tmp/icc-config.cfg
+
+ENV MANPATH=/usr/share/man:/usr/local/man
+ENV LD_LIBRARY_PATH=/usr/lib:/usr/lib64
+
+RUN ls /opt/intelicc/
+RUN mkdir /opt/intelicc/compilers_and_libraries_2020.0.166/linux/licenses/
+COPY x.lic /opt/intelicc/compilers_and_libraries_2020.0.166/linux/licenses/
+# set up workdir
+ENV COMPILERVARS_ARCHITECTURE='intel64'
+ENV COMPILERVARS_PLATFORM='linux'
+ENV ICC='source /opt/intelicc/bin/iccvars.sh'
+ENV MPIICC='source /opt/intelicc/bin/compilervars.sh'
+ENV CC=icc
+
+RUN yum clean all
+RUN rm -rf /opt/intelicc/advisor_2020.0.0.604394
+RUN rm -rf /opt/intelicc/vtune_profiler_2020.0.0.605129
+RUN rm -rf /opt/intelicc/inspector_2020.0.0.603904
+
+# set up base
+COPY ssh_config /root/.ssh/config
+RUN yum install -y ca-certificates wget openssh-server openssh-clients net-tools \
+        numactl-devel gtk2 atk cairo tcsh libnl3 tcl libmnl tk \
+        htop vim git-core which pciutils lsof ethtool gcc-gfortran
+
+# set up ssh keys
+RUN mkdir -p /var/run/sshd \
+    && ssh-keygen -A \
+    && sed -i 's/UsePAM yes/UsePAM no/g' /etc/ssh/sshd_config \
+    && sed -i 's/#PermitRootLogin yes/PermitRootLogin yes/g' /etc/ssh/sshd_config \
+    && sed -i 's/#RSAAuthentication yes/RSAAuthentication yes/g' /etc/ssh/sshd_config \
+    && sed -i 's/#PubkeyAuthentication yes/PubkeyAuthentication yes/g' /etc/ssh/sshd_config \
+    && ssh-keygen -f /root/.ssh/id_rsa -t rsa -N '' \
+    && chmod 600 /root/.ssh/config \
+    && chmod 700 /root/.ssh \
+    && cp /root/.ssh/id_rsa.pub /root/.ssh/authorized_keys
+
+# nompi vars
+ADD varsnompi.sh /opt/intelicc/bin/varsnompi.sh
+ENV ICC='source /opt/intelicc/bin/varsnompi.sh'
+
+
+##### MPI install
+# mlnx (IB required)
+ADD ./mlnx /tmp/mpi/mlnx
+RUN $ICC\
+    && F77=ifort FC=ifort CC=icc CXX=icpc /tmp/mpi/mlnx/mlnxofedinstall --user-space-only --without-fw-update --all --force \
+    && rm -rf /tmp/mpi/mlnx
+
+#  HPC-X (IB required)
+ADD ./hpcx /opt/hpcx
+ENV HPCX_HOME=/opt/hpcx
+
+RUN yum install -y systemd-devel libgudev1-devel libgudev1 zlib-devel zlib \
+                   libxml2 libxml2-devel binutils binutils-devel libpsm2-devel libpsm2 
+# OpenMPi from HPC-x
+
+# download and install MPICH
+RUN tar xfp ${HPCX_HOME}/sources/openmpi-gitclone.tar.gz
+RUN cd openmpi-gitclone \
+    && $ICC \
+    && source $HPCX_HOME/hpcx-init.sh && hpcx_load \
+    && CC=icc CXX=icpc F77=ifort FC=ifort ./configure \
+                   --with-ucx=$HPCX_HOME/ucx \
+                   --with-hcoll=/$HPCX_HOME/hcoll \
+                   --prefix=${HPCX_HOME}/ompi-icc \
+                   --with-platform=optimized \
+                   --enable-mca-no-build=btl-uct \
+                   --with-device=ch4:ucx 2>&1 | tee config-icc-output.log \
+    && make -j32 all 2>&1 | tee build_icc.log \
+    && make -j24 install 2>&1 | tee install_icc.log \
+    && cd .. \
+    && rm -rf openmpi-gitclone
+
+ENV OMPI_ALLOW_RUN_AS_ROOT=1
+ENV OMPI_ALLOW_RUN_AS_ROOT_CONFIRM=1
+
+# Devito
+RUN yum install -y python3 python3-pip python3-devel python3-libs
+RUN pip3 install --upgrade pip
+RUN git clone https://github.com/devitocodes/devito.git
+# RUN cd devito && $ICC && pip3 install --force-reinstall --no-cache-dir --upgrade -e .
+RUN cd devito && CC=gcc && pip3 install --force-reinstall --no-cache-dir --upgrade -e .
+RUN cd devito && $MPIICC && pip3 install mpi4py
+RUN $MPIICC && DEVITO_ARCH=intel python3 devito/examples/seismic/acoustic/acoustic_example.py
+
+#
+# Install storage queue
+RUN pip3 install azure-storage-blob h5py segyio
+RUN pip3 install azure-storage
+
+# Add application
+# ADD ./tti /app/tti
+# COPY setup-ompi.sh /opt/setup-ompi.sh
+
+# set up sshd on port 23
+EXPOSE 23
+CMD ["/usr/sbin/sshd", "-D", "-p", "23"]
diff --git a/recipes/devito-icc-intelmpi-drivers/docker/ssh_config b/recipes/devito-icc-intelmpi-drivers/docker/ssh_config
@@ -0,0 +1,4 @@
+Host *
+  Port 23
+  StrictHostKeyChecking no
+  UserKnownHostsFile /dev/null
diff --git a/recipes/devito-icc-intelmpi-drivers/shipyard-config/config.yaml b/recipes/devito-icc-intelmpi-drivers/shipyard-config/config.yaml
@@ -0,0 +1,13 @@
+batch_shipyard:
+  storage_account_settings: mystorageaccount
+global_resources:
+  docker_images:
+  - gbisbas/devitoicc:v0.8dhc44
+  volumes:
+    shared_data_volumes:
+      azureblob_vol:
+        volume_driver: azureblob
+        storage_account_settings: mystorageaccount
+        azure_blob_container_name: devito-bench
+        container_path: $AZ_BATCH_NODE_SHARED_DIR/bench-results
+        bind_options: rw
diff --git a/recipes/devito-icc-intelmpi-drivers/shipyard-config/jobs.yaml b/recipes/devito-icc-intelmpi-drivers/shipyard-config/jobs.yaml
@@ -0,0 +1,30 @@
+job_specifications:
+- id: bench-icc-intelmpi
+  shared_data_volumes:
+  - azureblob_vol
+  tasks:
+  - docker_image: gbisbas/devitoicc:v0.8dhc44
+    environment_variables:
+      DEVITO_ARCH: intel
+      OMP_NUM_THREADS: 22
+      DEVITO_MPI: 1
+      DEVITO_OPENMP: 1
+      KMP_AFFINITY: compact, granularity=core
+      DEVITO_LOGGING: DEBUG
+      # I_MPI_DEBUG: 2
+      I_MPI_PIN: 1
+      I_MPI_PIN_DOMAIN: sockets
+      # I_MPI_HYDRA_DEBUG: 1
+      FI_LOG_LEVEL: debug
+      LC_ALL: 'en_US.UTF-8'
+      UCX_TLS: ud,sm,self
+    default_working_dir: container
+    multi_instance:
+      num_instances: pool_current_dedicated
+      mpi:
+        runtime: intelmpi
+        processes_per_node: 1
+      pre_execution_command: source /opt/intelicc/compilers_and_libraries_2020.0.166/linux/bin/compilervars.sh -arch intel64 -platform linux; which icc; which mpicc; which mpirun; which mpiexec; ls;
+    command: python3 devito/examples/seismic/acoustic/acoustic_example.py
+    # IMB-MPI1 
+    # python3 devito/benchmarks/user/benchmark.py bench -P elastic --tn 400 -d 100 100 100 --resultsdir $AZ_BATCH_NODE_SHARED_DIR/results_bench --arch intel --dse advanced --dle advanced -x 1
diff --git a/recipes/devito-icc-intelmpi-drivers/shipyard-config/pool.yaml b/recipes/devito-icc-intelmpi-drivers/shipyard-config/pool.yaml
@@ -0,0 +1,15 @@
+pool_specification:
+  id: devito-drivers-icc-intelmpi
+  vm_configuration:
+    platform_image:
+      publisher: OpenLogic
+      offer: CentOS-HPC
+      sku: '7.6'
+  vm_count:
+    dedicated: 4
+    low_priority: 0
+  resize_timeout: 00:40:00
+  vm_size: Standard_HC44rs
+  inter_node_communication_enabled: true
+  ssh:
+    username: shipyard
diff --git a/recipes/devito-icc-intelmpi/docker/Dockerfile b/recipes/devito-icc-intelmpi/docker/Dockerfile
@@ -0,0 +1,54 @@
+FROM centos:7.6.1810
+RUN rm /bin/sh && ln -s /bin/bash /bin/sh
+
+# set up base and ssh keys
+COPY ssh_config /root/.ssh/config
+
+RUN yum swap -y fakesystemd systemd
+RUN yum -y update \
+    && yum install -y \
+        build-essential ca-certificates wget openssh-client numactl openssh-server curl gcc gcc-c++ systemd-devel ucx libgudev1-devel libgudev1 zlib-devel zlib libxml2 libxml2-devel binutils binutils-devel numactl-devel install libpsm2 libpsm2-devel librdmacm librdmacm-devel\
+    && yum clean all \
+    && mkdir -p /var/run/sshd \
+    && ssh-keygen -A \
+    && sed -i 's/UsePAM yes/UsePAM no/g' /etc/ssh/sshd_config \
+    && sed -i 's/#PermitRootLogin yes/PermitRootLogin yes/g' /etc/ssh/sshd_config \
+    && sed -i 's/#RSAAuthentication yes/RSAAuthentication yes/g' /etc/ssh/sshd_config \
+    && sed -i 's/#PubkeyAuthentication yes/PubkeyAuthentication yes/g' /etc/ssh/sshd_config \
+    && ssh-keygen -f /root/.ssh/id_rsa -t rsa -N '' \
+    && chmod 600 /root/.ssh/config \
+    && chmod 700 /root/.ssh \
+    && cp /root/.ssh/id_rsa.pub /root/.ssh/authorized_keys
+
+ENV DEBIAN_FRONTEND=noninteractive
+
+RUN yum install -y cpio libgtk2.0 xorg openbox
+COPY conf_mpi_test.cfg /tmp/icc-config.cfg
+ADD /parallel_studio_xe_2020_cluster_edition_online /tmp/intel
+RUN cd /tmp/intel && \
+   ./install.sh --silent=/tmp/icc-config.cfg && \
+   cd ../../ && \
+   rm -rf /tmp/intel && \
+   rm /tmp/icc-config.cfg
+
+RUN mkdir /opt/intelicc/compilers_and_libraries_2020.0.166/linux/licenses/
+COPY x.lic /opt/intelicc/compilers_and_libraries_2020.0.166/linux/licenses/
+RUN ls /opt/intelicc/compilers_and_libraries_2020.0.166/linux/licenses/
+
+RUN source /opt/intelicc/compilers_and_libraries/linux/bin/compilervars.sh -arch intel64 -platform linux && icc --help
+RUN source /opt/intelicc/compilers_and_libraries/linux/mpi/intel64/bin/mpivars.sh && mpirun --help
+
+# RUN ls /opt/intel/compilers_and_libraries/linux/mpi/intel64/bin/
+
+# Utils
+RUN yum install -y htop vim git-core
+
+# Devito
+RUN yum install -y python3 python3-pip python3-devel
+RUN pip3 install --upgrade pip
+RUN git clone https://github.com/devitocodes/devito.git
+RUN source /opt/intelicc/compilers_and_libraries/linux/mpi/intel64/bin/mpivars.sh && cd devito && pip3 install -e .[extras]
+
+# set up sshd on port 23
+EXPOSE 23
+CMD ["/usr/sbin/sshd", "-D", "-p", "23"]
diff --git a/recipes/devito-icc-intelmpi/docker/conf_mpi_test.cfg b/recipes/devito-icc-intelmpi/docker/conf_mpi_test.cfg
@@ -0,0 +1,19 @@
+ACCEPT_EULA=accept
+CONTINUE_WITH_OPTIONAL_ERROR=yes
+PSET_INSTALL_DIR=/opt/intel
+CONTINUE_WITH_INSTALLDIR_OVERWRITE=yes
+PSET_MODE=install
+ACTIVATION_SERIAL_NUMBER=xxxx-yyyyyyyy
+ACTIVATION_TYPE=serial_number
+AMPLIFIER_SAMPLING_DRIVER_INSTALL_TYPE=build
+AMPLIFIER_DRIVER_ACCESS_GROUP=vtune
+AMPLIFIER_DRIVER_PERMISSIONS=660
+AMPLIFIER_LOAD_DRIVER=yes
+AMPLIFIER_C_COMPILER=/usr/bin/gcc
+AMPLIFIER_KERNEL_SRC_DIR=/lib/modules/5.3.0-29-generic/build
+AMPLIFIER_MAKE_COMMAND=/usr/bin/make
+AMPLIFIER_INSTALL_BOOT_SCRIPT=yes
+AMPLIFIER_DRIVER_PER_USER_MODE=no
+INTEL_SW_IMPROVEMENT_PROGRAM_CONSENT=no
+ARCH_SELECTED=ALL
+COMPONENTS=;intel-conda-index-tool__x86_64;intel-comp__x86_64;intel-comp-32bit__x86_64;intel-comp-doc__noarch;intel-comp-l-all-common__noarch;intel-comp-l-all-vars__noarch;intel-comp-nomcu-vars__noarch;intel-comp-ps-32bit__x86_64;intel-comp-ps__x86_64;intel-comp-ps-ss-bec__x86_64;intel-comp-ps-ss-bec-32bit__x86_64;intel-openmp__x86_64;intel-openmp-32bit__x86_64;intel-openmp-common__noarch;intel-openmp-common-icc__noarch;intel-openmp-common-ifort__noarch;intel-openmp-ifort__x86_64;intel-openmp-ifort-32bit__x86_64;intel-tbb-libs-32bit__x86_64;intel-tbb-libs__x86_64;intel-idesupport-icc-common-ps__noarch;intel-conda-intel-openmp-linux-64-shadow-package__x86_64;intel-conda-intel-openmp-linux-32-shadow-package__x86_64;intel-conda-icc_rt-linux-64-shadow-package__x86_64;intel-icc__x86_64;intel-icc-32bit__x86_64;intel-c-comp-common__noarch;intel-icc-common__noarch;intel-icc-common-ps__noarch;intel-icc-doc__noarch;intel-icc-ps__x86_64;intel-icc-ps-ss-bec__x86_64;intel-icc-ps-ss-bec-32bit__x86_64;intel-icx__x86_64;intel-icx-common__noarch;intel-tbb-devel-32bit__x86_64;intel-tbb-devel__x86_64;intel-tbb-common__noarch;intel-tbb-doc__noarch;intel-conda-tbb-linux-64-shadow-package__x86_64;intel-conda-tbb-linux-32-shadow-package__x86_64;intel-conda-tbb-devel-linux-64-shadow-package__x86_64;intel-conda-tbb-devel-linux-32-shadow-package__x86_64;intel-imb__x86_64;intel-mpi-rt__x86_64;intel-mpi-sdk__x86_64;intel-mpi-doc__x86_64;intel-mpi-samples__x86_64;intel-conda-impi_rt-linux-64-shadow-package__x86_64;intel-conda-impi-devel-linux-64-shadow-package__x86_64;intel-gdb__x86_64;intel-gdb-source__noarch;intel-gdb-python-source__noarch;intel-gdb-common__noarch;intel-gdb-common-ps__noarch;intel-icsxe__noarch;intel-psxe-common__noarch;intel-psxe-doc__noarch;intel-psxe-common-doc__noarch;intel-icsxe-doc__noarch;intel-psxe-licensing__noarch;intel-psxe-licensing-doc__noarch;intel-icsxe-pset
diff --git a/recipes/devito-icc-intelmpi/docker/ssh_config b/recipes/devito-icc-intelmpi/docker/ssh_config
@@ -0,0 +1,4 @@
+Host *
+  Port 23
+  StrictHostKeyChecking no
+  UserKnownHostsFile /dev/null
diff --git a/recipes/devito-icc-intelmpi/shipyard-config/config.yaml b/recipes/devito-icc-intelmpi/shipyard-config/config.yaml
@@ -0,0 +1,13 @@
+batch_shipyard:
+  storage_account_settings: mystorageaccount
+global_resources:
+  docker_images:
+  - gbisbas/devitoicc:v0.7hc44
+  volumes:
+    shared_data_volumes:
+      azureblob_vol:
+        volume_driver: azureblob
+        storage_account_settings: mystorageaccount
+        azure_blob_container_name: devito-bench
+        container_path: $AZ_BATCH_NODE_SHARED_DIR/bench-results
+        bind_options: rw
diff --git a/recipes/devito-icc-intelmpi/shipyard-config/jobs.yaml b/recipes/devito-icc-intelmpi/shipyard-config/jobs.yaml
@@ -0,0 +1,30 @@
+job_specifications:
+- id: bench-icc-intelmpi
+  shared_data_volumes:
+  - azureblob_vol
+  tasks:
+  - docker_image: gbisbas/devitoicc:v0.7hc44
+    environment_variables:
+      DEVITO_ARCH: intel
+      OMP_NUM_THREADS: 22
+      DEVITO_MPI: full
+      DEVITO_OPENMP: 1
+      KMP_AFFINITY: compact, granularity=core
+      DEVITO_LOGGING: DEBUG
+      I_MPI_DEBUG: 2
+      I_MPI_PIN: 1
+      I_MPI_PIN_DOMAIN: sockets
+      I_MPI_HYDRA_DEBUG: 1
+      I_MPI_FABRICS: ofa
+      FI_LOG_LEVEL: debug
+      FI_PROVIDER: verbs
+      LC_ALL: en_US.utf8
+      UCX_TLS: ud,sm,self
+    default_working_dir: container
+    multi_instance:
+      num_instances: pool_current_dedicated
+      mpi:
+        runtime: intelmpi
+        processes_per_node: 1
+      pre_execution_command: source /opt/intelicc/compilers_and_libraries_2020.0.166/linux/bin/compilervars.sh -arch intel64 -platform linux; which icc; which mpicc; which mpirun; which mpiexec;
+    command: python3 devito/examples/seismic/acoustic/acoustic_example.py
diff --git a/recipes/devito-icc-intelmpi/shipyard-config/pool.yaml b/recipes/devito-icc-intelmpi/shipyard-config/pool.yaml
@@ -0,0 +1,15 @@
+pool_specification:
+  id: devito-icc-intelmpi
+  vm_configuration:
+    platform_image:
+      publisher: OpenLogic
+      offer: CentOS-HPC
+      sku: '7.6'
+  vm_count:
+    dedicated: 4
+    low_priority: 0
+  resize_timeout: 00:40:00
+  vm_size: Standard_HC44rs
+  inter_node_communication_enabled: true
+  ssh:
+    username: shipyard