Merge branch 'develop'

Author: CFGrote

.gitignore

@@ -15,3 +15,4 @@ doc/doxy_build/
 deploy_rsa
 deploy_doc_to_gh-pages.sh
 deploy_rsa.enc
+*.bak

.travis.yml

@@ -1,70 +1,72 @@
 language: python
 python:
-  - "2.7"
-  - "3.4"
   - "3.5"
+  - "3.6"
   - "nightly"
 
 sudo: required
 dist: trusty
 
 matrix:
-  allow_failures:
-    # not yet Python 3 compatible
-    - python: "3.4"
-    - python: "3.5"
-    # Python 3.6 "nightly" (upcoming)
-    - python: "nightly"
+    allow_failures:
+        - python: "nightly"
 
 env:
   global:
     - C_INCLUDE_PATH=/usr/lib/openmpi/include
-    - DEPLOY_DOCS_FOR_PYTHON: '2.7'
-    - TRIGGER_DOCKER_FOR_PYTHON: '2.7'
+    - WITH_CUDA: true
+    - DEPLOY_DOCS_FOR_PYTHON: '3.5'
+    - TRIGGER_DOCKER_FOR_PYTHON: '3.5'
+    - SIMEX_VERBOSE=MPI:PYTHON
 
 addons:
   apt:
     packages:
       - openmpi-bin
       - libopenmpi-dev
       - wget
-      - libhdf5-dev # non-MPI but with HL & CPP bindings
+      - libhdf5-dev
       - build-essential
       - libbz2-dev
       - libfftw3-dev
       - liblapack-dev
       - cmake
       - gfortran
       - unzip
-      - cython
+      - cython3
       - sphinx-common
-      - python-sphinx
-      - python-qt4
+      - python3-sphinx
       - libgsl0-dev
+      - llvm
+      - nvidia-cuda-dev
+      - nvidia-profiler
+      - nvidia-cuda-toolkit
 
 cache:
-  apt: true
+  apt: false
   directories:
     - $HOME/.cache/pip
     - $HOME/lib
 
 # install git lfs
 #   see https://github.com/travis-ci/travis-ci/issues/3634
 before_install:
-- mkdir -p $HOME/bin
-- wget https://github.com/github/git-lfs/releases/download/v1.1.2/git-lfs-linux-amd64-1.1.2.tar.gz
-- tar xvfz git-lfs-linux-amd64-1.1.2.tar.gz
-- mv git-lfs-1.1.2/git-lfs $HOME/bin/git-lfs
-- export PATH=$PATH:$HOME/bin/
-- bash $TRAVIS_BUILD_DIR/TravisCI/pyqt.sh
+    - mkdir -p $HOME/bin
+    - wget https://github.com/github/git-lfs/releases/download/v1.1.2/git-lfs-linux-amd64-1.1.2.tar.gz
+    - tar xvfz git-lfs-linux-amd64-1.1.2.tar.gz
+    - mv git-lfs-1.1.2/git-lfs $HOME/bin/git-lfs
+    - export PATH=$PATH:$HOME/bin/
+      #- bash $TRAVIS_BUILD_DIR/TravisCI/pyqt.sh ## only needed for python v<3.5
+
 before_script:
-  # print interesting versions
   - python --version
   - python -c "import numpy as np; print( np.__version__ )"
   - python -c "import h5py as h5; print( h5.__version__ )"
   - ulimit -c unlimited
   - mpirun --version
+
 script:
+    #- wget ftp://ftp.wwpdb.org/pub/pdb/data/structures/divided/pdb/ni/pdb2nip.ent.gz
   - bash $TRAVIS_BUILD_DIR/TravisCI/build_and_test_simex.sh && bash $TRAVIS_BUILD_DIR/TravisCI/deploy_docs.sh
 after_success:
   - bash $TRAVIS_BUILD_DIR/TravisCI/trigger_docker.sh

CMakeLists.txt

@@ -19,42 +19,21 @@ option (DEVELOPER_INSTALL "Install modules only, do not install SimEx files" OFF
 
 option (INSTALL_TESTS "Install tests" ON)
 
-
 # options required for building the py_detector_interface module:
 # XCSITPotonDetector
-option(GEANT4_ROOT "-DGEANT4_ROOT=<abs path> absolute path to the geant4
-    installation directory that includes \n
-    ./bin/\n
-    ./include/Geant4/\n
-    ./lib64/Geant4-<ver>/\n
-    ./share/Geant4-<ver>/data/"
-)
-option(XERCESC_ROOT "-DXERCESC_ROOT=<abs path> absolute path to the xercesc
-    installation directory that includes\n
-    ./include/xercesc/\n
-    ./lib/"
-)
-option(XCSIT_ROOT "-DXCSIT_ROOT=<abs path> absolute path to the xcist
-    installation directory that includes\n
-    ./include/xcsit/
-    ./lib/
-    ./lib/xcsit/
-    ./bin/"
-)
-option(BOOST_ROOT "-DBOOST_ROOT=<abs path> absolute path to the boost
-    installation directory that includes\n
-    ./include/boost/
-    ./lib/"
-)
 option(PY_DETECTOR_INTERFACE_DEBUG "
     -DPY_DETECTOR_INTERFACE_DEBUG=default: OFF | ON"
 )
 
+### PYTHON
+FIND_PACKAGE(PythonInterp 3.5 REQUIRED)
 
 set (LIBDIR     lib)
 set (INCLUDEDIR include)
 set (BINDIR     bin)
-set (PYPATH     ${LIBDIR}/python2.7)
+set (PYPATH     ${LIBDIR}/python${PYTHON_VERSION_MAJOR}.${PYTHON_VERSION_MINOR})
+message( "PYPATH=${PYPATH}" )
+
 set (DOCDIR share/doc)
 
 option (PACKAGE_MAKE "Use this option to create linux packages" OFF)
@@ -77,25 +56,57 @@ else()
   endif()
 endif()
 
-  message (STATUS "************************ checking python dependencies *****************************")
-  add_subdirectory (Sources)
-
+message (STATUS "************************ checking python dependencies *****************************")
+add_subdirectory (Sources)
+
+#By default all modules are included/excluded depending on this flag.
+#If a flag: USE_${module_name} is passed, this will overwrite teh default.
+#For example:
+# [De-]activate all modules:
+#      cmake -DUSE_MODULES_DEFAULT=[OFF]ON [...]
+# Activate a single module:
+#      cmake -DUSE_MODULES_DEFAULT=OFF -DUSE_wpg=ON [...]
+# Deactivate a single module:
+#      cmake -DUSE_MODULES_DEFAULT=ON -DUSE_wpg=OFF [...]
+option(USE_MODULES_DEFAULT "Default to include/exclude a module" ON)
+message(STATUS "**************** Including modules (default "
+            "${USE_MODULES_DEFAULT}) ****************")
 # find and add modules
 set (SIMEX_MODULES_ROOT "${SIMEX_SOURCE_DIR}/Modules")
 include (find_modules)
 foreach(module ${SIMEX_MODULES})
     MESSAGE("CHECKING ${module}")
     get_filename_component(module_name ${module} NAME)
-    option(${module_name} "include ${module_name} module" ON)
+    #If a variable USE_${module_name} is defined, use its value
+    #overwrite the default.
+    #Since we add the USE_* variable to the cache, we re-use it with
+    #multiple cmake calls
+    if(DEFINED USE_${module_name})
+        #Add a cache variable with some help text. Note the
+        #use of FORCE option to be sure the value is updated in the
+        #cache
+        set(USE_${module_name} ${USE_${module_name}}
+                CACHE BOOL "Include mdoule ${module_name}" FORCE)
+    endif()
 endforeach()
 
 foreach(module ${SIMEX_MODULES})
     get_filename_component(module_name ${module} NAME)
-    if (${module_name})
-	    message (STATUS "***** Including module : ${module} ****")
-        add_subdirectory("${SIMEX_MODULES_ROOT}/${module}")
-	    message (STATUS "*********************************************************")
+    #If a request specific to this module was done, use it
+    if (DEFINED USE_${module_name})
+        if(USE_${module_name})
+            message (STATUS "***** Including module ${module_name} from: ${module} ****")
+            add_subdirectory("${SIMEX_MODULES_ROOT}/${module}")
+	        message (STATUS "*********************************************************")
+        endif()
+    else()
+        if(USE_MODULES_DEFAULT)
+            message (STATUS "***** Including module ${module_name} from: ${module} ****")
+            add_subdirectory("${SIMEX_MODULES_ROOT}/${module}")
+	        message (STATUS "*********************************************************")
+        endif()
 	endif()
+
 endforeach()
 
 # create simex_vars.sh file

Changelog.rst

@@ -1,5 +1,15 @@
 CHANGELOG
 =========
+Changes from 0.3 to 0.4
+-----------------------
+    * Debugged XCSITPhotonDetector
+    * Added phase grating simulation to WPG (wavefront propagation)
+    * Added SASE model for low gain FEL via ocelot
+    * GAPDPhotonDiffractor for polychromatic diffraction from large samples (>10^5 atoms. (GAPD code).
+    * Many new data analysis utilities
+    * Integration of python-pint to support pysical units and constants
+    * New XMDYNPhotonMatterInteractor to wrap XMDYN photon-matter interaction code
+    * New SLURM Submitter
 
 Changes from 0.2 to 0.3
 -----------------------

Docker/simex/Dockerfile

@@ -1,12 +1,19 @@
-FROM yakser/simex_3rdparty
-MAINTAINER Sergey Yakubov <[email protected]>
+FROM chuckie82/centos_env
+MAINTAINER Chunhong Yoon <[email protected]>
+
+ADD requirements.txt /opt/requirements.txt
+ADD python_install.sh /opt/python_install.sh
+RUN ["bash", "/opt/python_install.sh"]
+ENV PATH /opt/miniconda/bin:$PATH
 
 ARG simex_script=simex_install.sh
 ADD $simex_script /opt/simex_install.sh
+
 RUN ["bash", "/opt/simex_install.sh"]
+
 ENV SIMEX_ROOT=/opt/simex_platform
 ENV PATH=$SIMEX_ROOT/bin:$PATH
-ENV PYTHONPATH=$SIMEX_ROOT/Sources/python:$SIMEX_ROOT/lib/python2.7:$PYTHONPATH
+ENV PYTHONPATH=$SIMEX_ROOT/Sources/python:$SIMEX_ROOT/lib/python3.6:$PYTHONPATH
 ENV SIMEX_TESTS=$SIMEX_ROOT/Tests
 ENV PYFAI_TESTIMAGES=/tmp
-ENTRYPOINT ["python"]
+

Docker/simex/build_image

@@ -1,2 +1,4 @@
-docker build --no-cache -t yakser/simex .
+cp ../../requirements.txt .
+
+docker build --no-cache -t chuckie82/simex .
 

Docker/simex/python_install.sh

@@ -0,0 +1,47 @@
+#!/usr/bin/env bash
+
+export myminiconda=Miniconda3-4.4.10-Linux-x86_64.sh
+URL=https://repo.continuum.io/miniconda/$myminiconda
+
+# Install dir
+PREFIX=/opt/miniconda
+
+# Download Miniconda
+wget -q $URL
+
+# Extract packages
+
+chmod +x $myminiconda
+./$myminiconda -b -p $PREFIX
+rm ./$myminiconda
+cd $PREFIX
+
+# config
+export PATH=$PREFIX/bin:$PATH
+conda config --set always_yes True
+
+# pip cannot install pyqt
+conda install --no-update-deps pyqt=4
+
+# delete tests
+find . -type d -name tests -depth -exec rm -rf {} \;
+find . -type d -name test -depth -exec rm -rf {} \;
+
+conda clean --tarballs
+
+# install requirements from file (in given order)
+#$PREFIX/bin/pip install -r /opt/requirements.txt
+xargs -L 1 pip install < /opt/requirements.txt
+
+# remove .pyc
+find . -name \__pycache__ -depth -exec rm -rf {} \;
+find . -name "*.pyc" -exec rm -rf {} \;
+
+
+# remove cache
+rm -r pkgs/*
+rm -rf /root/.cache/pip
+
+
+echo "export PATH=/opt/miniconda/bin:${PATH}" >> /etc/profile.d/scripts-path.sh
+

Docker/simex/simex_install.sh

@@ -5,19 +5,25 @@ set -e
 BRANCH=develop
 URL=https://github.com/eucall-software/simex_platform/archive/${BRANCH}.zip
 
-
 cd /opt
 
 wget $URL
 unzip ${BRANCH}.zip
 rm ${BRANCH}.zip
 cd simex_platform-${BRANCH}
 
-export PATH=/opt/miniconda2/bin:$PATH
-export HDF5_ROOT=/opt/miniconda2
+export PATH=/opt/miniconda/bin:$PATH
+export HDF5_ROOT=/opt/miniconda
+
+echo "###### DONE unzip ${BRANCH}.zip"
+
+conda install -c intel mkl
+
+echo "##### DONE install mkl"
 
 ROOT_DIR=/opt/simex_platform
 mkdir -p $ROOT_DIR
+
 # Create new build dir and cd into it.
 
 mkdir -v build
@@ -26,23 +32,34 @@ cd build
 # Uncomment the next line and specify the install dir for a custom user install.
 #cmake -DCMAKE_INSTALL_PREFIX=$ROOT_DIR $ROOT_DIR
 # Uncomment the next line and specify the install dir for a developer install.
-cmake -DINSTALL_TESTS=OFF -DSRW_OPTIMIZED=ON -DDEVELOPER_INSTALL=OFF -DCMAKE_INSTALL_PREFIX=$ROOT_DIR $ROOT_DIR ..
+cmake -DXCSITPhotonDetector=OFF \
+      -DGAPDPhotonDiffractor=OFF \
+      -DCrystFELPhotonDiffractor=OFF \
+      -DINSTALL_TESTS=OFF \
+      -DSRW_OPTIMIZED=ON \
+      -DDEVELOPER_INSTALL=OFF \
+      -DCMAKE_INSTALL_PREFIX=$ROOT_DIR \
+      $ROOT_DIR \
+      ..
 
 chmod og+rwX -R $ROOT_DIR
 
 # Build the project.
 make
 
+echo "######## done make"
 
 # Install the project.
 make install
 cd ../..
 
-rm -rf simex_platform-${BRANCH}
+echo "####### done make install"
+
+#rm -rf simex_platform-${BRANCH}
 
 
 #remove tests?
-rm -rf $ROOT_DIR/Tests
+#rm -rf $ROOT_DIR/Tests
 
 echo "source /opt/simex_platform/bin/simex_vars.sh" > /etc/profile.d/scripts-simex.sh && \
 	chmod 755 /etc/profile.d/scripts-simex.sh

Modules/Analyzers/S2EReconstruction_DM/makefile_DM

@@ -1,5 +1,5 @@
 default:
-	gcc -O3 object_recon.c -lm -Wno-unused-result -lfftw3 -o object_recon
+	gcc -O3 object_recon.c -Wno-unused-result -fopenmp -lfftw3_omp -lfftw3 -lm -o object_recon
 mkl:
-	gcc -O3 object_recon.c -o object_recon -m64 -I${MKLROOT}/include -I${MKLROOT}/include/fftw -Wl,--no-as-needed -L${MKLROOT}/lib/intel64 -lmkl_intel_lp64 -lmkl_core -lmkl_sequential -lpthread -lm
+	gcc -O3 object_recon.c -o object_recon -m64 -I${MKLROOT}/include -I${MKLROOT}/include/fftw -Wl,--no-as-needed -L${MKLROOT}/lib/intel64 -lmkl_intel_lp64 -lmkl_core -lmkl_sequential -liomp5 -lpthread -lm -ldl