Merge pull request #228 from PaNOSC-ViNYL/develop

pre-release v0.5.0-alpha

Author: JunCEEE

.gitignore

@@ -1,6 +1,6 @@
 *.pyc
 *.pyo
-**.swp
+.*.*.swp
 bin
 include/
 !Sources/doc/source/include
@@ -18,3 +18,6 @@ deploy_rsa.enc
 *.bak
 *.code-workspace
 *.root
+.ropeproject
+tags
+Tests/python/unittest/TestFiles/

.travis.yml

@@ -1,20 +1,26 @@
 language: python
 python:
+  - "3.5"
   - "3.6"
+  - "3.7"
+  - "3.8"
+  - "nightly"
 
 sudo: required
-dist: trusty
+dist: xenial
 
 matrix:
     allow_failures:
+        - python: "3.5"
+        - python: "3.6"
         - python: "nightly"
 
 env:
   global:
     - C_INCLUDE_PATH=/usr/lib/openmpi/include
     - WITH_CUDA: true
-    - DEPLOY_DOCS_FOR_PYTHON: '3.5'
-    - TRIGGER_DOCKER_FOR_PYTHON: '3.5'
+    - DEPLOY_DOCS_FOR_PYTHON: '3.8'
+    - TRIGGER_DOCKER_FOR_PYTHON: '3.8'
     - SIMEX_VERBOSE=MPI:PYTHON
 
 addons:
@@ -50,11 +56,8 @@ cache:
 #   see https://github.com/travis-ci/travis-ci/issues/3634
 before_install:
     - mkdir -p $HOME/bin
-    - wget https://github.com/github/git-lfs/releases/download/v1.1.2/git-lfs-linux-amd64-1.1.2.tar.gz
-    - tar xvfz git-lfs-linux-amd64-1.1.2.tar.gz
-    - mv git-lfs-1.1.2/git-lfs $HOME/bin/git-lfs
-    - export PATH=$PATH:$HOME/bin/
-      #- bash $TRAVIS_BUILD_DIR/TravisCI/pyqt.sh ## only needed for python v<3.5
+    - export Python_ROOT_DIR=$(dirname $(dirname $(which python)))
+
 
 before_script:
   - python --version

CMake/FindPythonLibs.cmake

@@ -90,7 +90,7 @@ set(CMAKE_FIND_FRAMEWORK LAST)
 
 set(_PYTHON1_VERSIONS 1.6 1.5)
 set(_PYTHON2_VERSIONS 2.7 2.6 2.5 2.4 2.3 2.2 2.1 2.0)
-set(_PYTHON3_VERSIONS 3.6 3.5 3.4 3.3 3.2 3.1 3.0)
+set(_PYTHON3_VERSIONS 3.7 3.6 3.5 3.4 3.3 3.2 3.1 3.0)
 
 if(PythonLibs_FIND_VERSION)
     if(PythonLibs_FIND_VERSION_COUNT GREATER 1)

CMakeLists.txt

@@ -1,4 +1,4 @@
-cmake_minimum_required (VERSION 2.8)
+cmake_minimum_required (VERSION 3.12)
 
 project (SIMEX)
 
@@ -8,35 +8,38 @@ include(CMakeDependentOption)
 if ("${SIMEX_SOURCE_DIR}"  STREQUAL "${SIMEX_BINARY_DIR}")
   message(FATAL_ERROR
     "SIMEX requires an out of source Build. "
-    "Please create a separate binary directory and run CMake there.")
+    "Please create a separate build/ directory and run CMake there.")
 endif()
 
 # set path for modules
 set (SIMEX_CMAKE_DIR "${SIMEX_SOURCE_DIR}/CMake")
 set (CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} ${SIMEX_CMAKE_DIR})
 
+# Build variant.
 option (DEVELOPER_INSTALL "Install modules only, do not install SimEx files" OFF)
-
 option (INSTALL_TESTS "Install tests" ON)
+option (PACKAGE_MAKE "Use this option to create linux packages" OFF)
 
-# options required for building the py_detector_interface module:
+# Options required for building the py_detector_interface module:
 # XCSITPotonDetector
 option(PY_DETECTOR_INTERFACE_DEBUG "
     -DPY_DETECTOR_INTERFACE_DEBUG=default: OFF | ON"
 )
 
 ### PYTHON
-FIND_PACKAGE(PythonInterp 3.5 REQUIRED)
+FIND_PACKAGE(Python
+             3.5
+             REQUIRED
+             )
 
 set (LIBDIR     lib)
 set (INCLUDEDIR include)
 set (BINDIR     bin)
-set (PYPATH     ${LIBDIR}/python${PYTHON_VERSION_MAJOR}.${PYTHON_VERSION_MINOR})
+set (PYPATH     ${LIBDIR}/python${Python_VERSION_MAJOR}.${Python_VERSION_MINOR}/site-packages)
 message( "PYPATH=${PYPATH}" )
 
 set (DOCDIR share/doc)
 
-option (PACKAGE_MAKE "Use this option to create linux packages" OFF)
 if (PACKAGE_MAKE)
  set (TESTSDIR   share/simex/Tests)
 else()
@@ -60,15 +63,15 @@ message (STATUS "************************ checking python dependencies *********
 add_subdirectory (Sources)
 
 #By default all modules are included/excluded depending on this flag.
-#If a flag: USE_${module_name} is passed, this will overwrite teh default.
+#If a flag: USE_${module_name} is passed, this will overwrite the default.
 #For example:
 # [De-]activate all modules:
 #      cmake -DUSE_MODULES_DEFAULT=[OFF]ON [...]
 # Activate a single module:
 #      cmake -DUSE_MODULES_DEFAULT=OFF -DUSE_wpg=ON [...]
 # Deactivate a single module:
 #      cmake -DUSE_MODULES_DEFAULT=ON -DUSE_wpg=OFF [...]
-option(USE_MODULES_DEFAULT "Default to include/exclude a module" ON)
+option(USE_MODULES_DEFAULT "Default to include/exclude all modules" OFF)
 message(STATUS "**************** Including modules (default "
             "${USE_MODULES_DEFAULT}) ****************")
 # find and add modules
@@ -137,6 +140,3 @@ endif()
 
 # Add documentation target.
 add_subdirectory("Sources/doc")
-
-# Add extra stuff, e.g. python libs not pip'able.
-add_subdirectory("Sources/python/extra")

Docker/simex/build_image

@@ -1,4 +1,4 @@
 cp ../../requirements.txt .
 
-docker build --no-cache -t chuckie82/simex .
+docker build --no-cache -t cfgrote/simex .
 

Docker/simex/simex_install.sh

@@ -3,7 +3,7 @@
 set -e
 
 BRANCH=develop
-URL=https://github.com/eucall-software/simex_platform/archive/${BRANCH}.zip
+URL=https://github.com/PaNOSC-ViNYL/SimEx/archive/${BRANCH}.zip
 
 cd /opt
 

Docker/simex_devel/simex_install_devel.sh

@@ -4,7 +4,7 @@ set -e
 
 
 BRANCH=develop
-URL=https://github.com/eucall-software/simex_platform.git
+URL=https://github.com/PaNOSC-ViNYL/SimEx.git
 cd /opt
 
 wget https://github.com/github/git-lfs/releases/download/v1.1.2/git-lfs-linux-amd64-1.1.2.tar.gz

Modules/Diffractors/CrystFELPhotonDiffractor/CMakeLists.txt

@@ -18,7 +18,9 @@ MESSAGE("HDF5_VERSION= ${HDF5_VERSION}")
 MESSAGE("HDF5_LIBRARY_DIRS= ${HDF5_LIBRARY_DIRS}")
 
 FIND_PACKAGE (GSL REQUIRED)
-
+find_package(ZLIB)
+include_directories(${ZLIB_INCLUDE_DIRS})
+# TARGET_LINK_LIBRARIES( ${ZLIB_LIBRARIES})
 # This macro invokes the HDF5 compiler wrapper, it parses the output and
 # extracts the path where library are stored.
 # Usage:
@@ -97,17 +99,18 @@ IF(${_listlen} GREATER 1)
 ENDIF()
 # Add the project.
 ExternalProject_Add( crystfel
-    URL http://www.desy.de/~twhite/crystfel/crystfel-0.6.3.tar.gz
-    BUILD_IN_SOURCE 1
-    #CONFIGURE_COMMAND ./configure --with-hdf5=${MYHDF5_LIBRARY_DIRS} --disable-gtk --disable-gtk-doc --disable-png
-    CONFIGURE_COMMAND ./configure --with-hdf5=/usr/lib64 --disable-gtk --disable-gtk-doc --disable-png --prefix=${CMAKE_INSTALL_PREFIX}
+    URL http://www.desy.de/~twhite/crystfel/crystfel-0.9.1.tar.gz
+    BUILD_IN_SOURCE 0
+    INSTALL_COMMAND ""
 )
 
 # Install needed crystfel binaries, scripts and doc material.
+ExternalProject_Get_Property(crystfel BINARY_DIR)
 ExternalProject_Get_Property(crystfel SOURCE_DIR)
+ExternalProject_Get_Property(crystfel BINARY_DIR)
 #
-#install(PROGRAMS ${SOURCE_DIR}/src/pattern_sim DESTINATION ${BINDIR})
-#install(PROGRAMS ${SOURCE_DIR}/src/partial_sim DESTINATION ${BINDIR})
+install(PROGRAMS ${BINARY_DIR}/pattern_sim DESTINATION ${BINDIR})
+install(PROGRAMS ${BINARY_DIR}/partial_sim DESTINATION ${BINDIR})
 install(PROGRAMS ${SOURCE_DIR}/scripts/gen-sfs DESTINATION ${BINDIR})
 install(PROGRAMS ${SOURCE_DIR}/scripts/gen-sfs-ano DESTINATION ${BINDIR})
 install(PROGRAMS ${SOURCE_DIR}/scripts/gen-sfs-expand DESTINATION ${BINDIR})

Modules/Others/sdf/CMakeLists.txt

@@ -0,0 +1,9 @@
+INCLUDE(ExternalProject)
+
+ExternalProject_Add(
+sdf
+GIT_REPOSITORY https://cfsa-pmw.warwick.ac.uk/SDF/SDF.git
+GIT_TAG 642be14ccae3bc7db53995f3fa8ea831bbbd3c6a
+GIT_SUBMODULES C utilities
+CMAKE_ARGS -DCMAKE_INSTALL_PREFIX=${PYPATH}
+)

Modules/Propagators/XFELPhotonPropagator/wpg/CMakeLists.txt Modules/Propagators/WavePropagator/wpg/CMakeLists.txt

@@ -4,12 +4,13 @@ option (SRW_OPTIMIZED "Use optimized SRW library" TRUE)
 
 ExternalProject_Add(
 wpg
-URL  https://github.com/eucall-software/WPG/archive/feature/openmp.zip
+# URL  https://github.com/PaNOSC-ViNYL/WPG/archive/develop.zip
+URL  https://github.com/samoylv/WPG/archive/develop.zip
 PATCH_COMMAND ""
 CONFIGURE_COMMAND ""
 CMAKE_COMMAND ""
 BUILD_IN_SOURCE 1
-BUILD_COMMAND make all
+BUILD_COMMAND OPENMP=omp make all
 INSTALL_COMMAND ""
 )
 

Modules/Propagators/XFELPhotonPropagator/wpg/Makefile.patch Modules/Propagators/WavePropagator/wpg/Makefile.patch

@@ -3,7 +3,8 @@ INCLUDE(ExternalProject)
 ExternalProject_Add(
 ocelot
 #URL https://github.com/eucall-software/ocelot/archive/dev_gen.zip
-URL https://github.com/ocelot-collab/ocelot/archive/dev_gen.zip
+#URL https://github.com/ocelot-collab/ocelot/archive/dev_gen.zip
+URL https://github.com/panosc-vinyl/ocelot/archive/dev_gen.zip
 PATCH_COMMAND ""
 INSTALL_COMMAND ""
 CONFIGURE_COMMAND ""

Modules/Propagators/XFELPhotonPropagator/wpg/genesis.py.patch Modules/Propagators/WavePropagator/wpg/genesis.py.patch

@@ -1,17 +1,15 @@
-SIMEX PLATFORM
+SimEx
 ==================
 
-Software platform for simulation of advanced photon experiments.
+Software platform for **Sim**ulation of advanced photon **Ex**periments.
 
-[![Build Status](https://travis-ci.org/PaNOSC-ViNYL/SimEx.svg?branch=master)](https://travis-ci.org/PaNOSC-ViNYL/SimEx)
-[![Build Status](https://travis-ci.org/PaNOSC-ViNYL/SimEx.svg?branch=develop)](https://travis-ci.org/PaNOSC-ViNYL/SimEx)
+[![Build Status master](https://travis-ci.org/PaNOSC-ViNYL/SimEx.svg?branch=master)](https://travis-ci.org/github/panosc-vinyl/simex/branches)
+[![Build Status develop](https://travis-ci.org/PaNOSC-ViNYL/SimEx.svg?branch=develop)](https://travis-ci.org/github/panosc-vinyl/simex/branches)
 
 
+## Purpose of SimEx
 
-
-## Purpose of simex_platform
-
-simex_platform is a python library to facilitate setup, execution, and analysis of
+SimEx is a python library to facilitate setup, execution, and analysis of
 simulations of experiments at advanced laser light sources.
 As an example, consider a molecule radiated by highly brilliant,
 ultrashort x-ray pulses such as delivered by an X-Ray Free Electron Laser (X-FEL).
@@ -23,7 +21,18 @@ field and detection of the latter. The platform provides slots and
 interfaces for the various simulation steps.
 
 For more details (User Manual, installation instructions, examples, etc.),
-please visit the project's homepage at [https://eucall-software.github.io/simex_platform/](https://eucall-software.github.io/simex_platform/)
+please visit the project's homepage at https://panosc-vinyl.github.io/SimEx/
+
+## Development
+
+The size of this Github repository of SimEx is huge due to the historical TestFiles.
+
+This command can clone only the newest develop branch to reduce the dowloading size:
+`git clone --depth 1 -b develop [email protected]:PaNOSC-ViNYL/SimEx.git`
+
+Now the TestFiles are hosted at [Zenodo](https://zenodo.org/record/3750541#.X2R9DZMzZE5).
+The files can be downloaded with [this script](get_testdata.sh).
 
 ## Acknowledgements
-This project has received funding from the European Union’s Horizon 2020 research and innovation programme under grant agreement No 654220.
+This project has received funding from the European Union’s Horizon 2020 research
+and innovation programme under grant agreement No 654220 and No 823852.

Modules/Propagators/XFELPhotonPropagator/wpg/glossary.py.patch Modules/Propagators/WavePropagator/wpg/glossary.py.patch

@@ -1,27 +1,19 @@
 # find needed packages
 include(FindPythonModules)
+find_python_module(Bio REQUIRED)
+find_python_module(Cython REQUIRED)
+find_python_module(dill REQUIRED)
 find_python_module(h5py REQUIRED)
 find_python_module(matplotlib REQUIRED)
-find_python_module(scipy REQUIRED)
 find_python_module(mpi4py REQUIRED)
-find_python_module(periodictable REQUIRED)
-find_python_module(dill REQUIRED)
-find_python_module(Bio REQUIRED)
-find_python_module(argparse REQUIRED)
-find_python_module(Cython REQUIRED)
-find_python_module(datetime REQUIRED)
+find_python_module(openpmd_api REQUIRED)
 find_python_module(numpy REQUIRED)
-find_python_module(setuptools REQUIRED)
-find_python_module(pyFAI REQUIRED)
-find_python_module(numba REQUIRED)
+find_python_module(periodictable REQUIRED)
 find_python_module(pint REQUIRED)
-find_python_module(breathe REQUIRED)
-find_python_module(sqlalchemy REQUIRED)
-find_python_module(pyfftw REQUIRED)
-#find_python_module(PyQt4 REQUIRED)
-
-#FIND_PACKAGE (LAPACK REQUIRED)
-#message (STATUS "***** LAPACK_LIBRARIES ${LAPACK_LIBRARIES} ****")
+find_python_module(pyFAI REQUIRED)
+find_python_module(scipy REQUIRED)
+find_python_module(setuptools REQUIRED)
+find_python_module(xraydb REQUIRED)
 
 if (NOT DEVELOPER_INSTALL)
 	install(DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/python/  DESTINATION ${PYPATH}

Modules/Sources/GenesisPhotonSource/CMakeLists.txt

@@ -1,3 +1,6 @@
+include(FindPythonModules)
+find_python_module(breathe REQUIRED)
+
 INCLUDE(ExternalProject)
 
 #soft find package to set needed variable