Use validation files for CI

.github/workflows/ls1_test.yml

@@ -62,6 +62,8 @@ jobs:
           libopenmpi-dev        \
           libomp-dev            \
           libxerces-c-dev
+        # Install python modules to execute validation script
+        pip3 install numpy pandas
         echo "Running ${JOBNAME}"
         git status
         mkdir build_${JOBNAME}
@@ -84,16 +86,23 @@ jobs:
             alllbl_enabled='OFF'
           fi
 
-          cmake -DVECTOR_INSTRUCTIONS=${{ matrix.vector }} \
-                -DCMAKE_BUILD_TYPE=${{ matrix.target }} \
-                -DENABLE_AUTOPAS=${{ matrix.autopas }} \
-                -DAUTOPAS_ENABLE_RULES_BASED_AND_FUZZY_TUNING=ON \
-                -DENABLE_ALLLBL=$alllbl_enabled \
-                -DOPENMP=${{ matrix.openmp }} \
-                -DENABLE_MPI=$mpi_enabled \
-                -DENABLE_UNIT_TESTS=ON \
-                -DENABLE_VTK=ON \
-              ..
+          cmake_exec="cmake -DVECTOR_INSTRUCTIONS=${{ matrix.vector }} \
+            -DCMAKE_BUILD_TYPE=${{ matrix.target }} \
+            -DENABLE_AUTOPAS=${{ matrix.autopas }} \
+            -DAUTOPAS_ENABLE_RULES_BASED_AND_FUZZY_TUNING=ON \
+            -DENABLE_ALLLBL=$alllbl_enabled \
+            -DOPENMP=${{ matrix.openmp }} \
+            -DENABLE_MPI=$mpi_enabled \
+            -DENABLE_UNIT_TESTS=ON \
+            -DENABLE_VTK=ON \
+            .."
+
+          # Omit quotes to trim whitespace of cmake_exec in second echo command
+          echo "The following cmake command is executed:"
+          echo CC=${{ matrix.cc }} CXX=${{ matrix.cxx }} ${cmake_exec}
+          echo ""
+
+          ${cmake_exec}
 
           cmake --build . --parallel 1
 
@@ -115,60 +124,23 @@ jobs:
     - if: ${{ matrix.parall == 'PAR' }}
       name: Validation
       run: |
+          #set strict pipefail option
+          set -eo pipefail
+
           #save absolute path to root of ls1 directory
           repoPath=$PWD
+
+          #example list of new version is used
           examplesFile="branchExamples_${JOBNAME}.txt"
-          # choose and save examples (so that this commit and master execute the same list)
+          # choose and save examples (so that the same file name could be used regardless of AutoPas is on or off)
           if [[ ${{ matrix.autopas }} == 'ON' ]]
           then
             cp ./examples/example-list_autopas.txt "${examplesFile}"
           else
             cp ./examples/example-list.txt "${examplesFile}"
           fi
-
-          #translate matrix to ON/OFF for certain entries
-          if [[ ${{ matrix.parall }} == 'PAR' ]]
-          then
-            mpi_enabled='ON'
-          else
-            mpi_enabled='OFF'
-          fi
-          if [[ ${{ matrix.parall }} == 'PAR' ]] && [[ ${{ matrix.autopas }} == 'ON' ]]
-          then
-            alllbl_enabled='ON'
-          else
-            alllbl_enabled='OFF'
-          fi
-
-          #build master branch equivalent to compare new build to
-          mkdir build_${JOBNAME}_master
-          git fetch
-          git checkout master
-          git status
-          cd build_${JOBNAME}_master
-          #note: ALLLBL is enabled if AutoPas is enabled.
-          cmake -DVECTOR_INSTRUCTIONS=${{ matrix.vector }} \
-                -DCMAKE_BUILD_TYPE=${{ matrix.target }} \
-                -DENABLE_AUTOPAS=${{ matrix.autopas }} \
-                -DAUTOPAS_ENABLE_RULES_BASED_AND_FUZZY_TUNING=ON \
-                -DENABLE_ALLLBL=$alllbl_enabled \
-                -DOPENMP=${{ matrix.openmp }} \
-                -DENABLE_MPI=$mpi_enabled \
-                -DENABLE_VTK=ON \
-              ..
 
-          cmake --build . --parallel 1
-
-          #as example list of new version is used, also the example files of new version should be used
-          #therefore, go back to new version
-          git checkout -
-
-          cd "${repoPath}"
-
-          #set strict pipefail option
-          set -eo pipefail
-
-          # execute all examples. These calls create artifacts which we will then compare
+          # execute all examples. These calls create artifacts which we will then compare to the validation JSON file
           IFS=$'\n'
           for i in $(cat "${repoPath}/${examplesFile}" )
           do
@@ -177,7 +149,7 @@ jobs:
             then
               continue
             fi
-            echo $i
+
             cd $repoPath/examples/$(dirname $i)
 
             # patch input files according to current conf
@@ -194,27 +166,22 @@ jobs:
               sed --in-place 's|AutoPas">|AutoPas">\n\t<functor>autoVec</functor>|g' input_patched.xml
             fi
 
-            # run the example with the old and new exe
-            for VERSION in "master" "new"
-            do
-              if [[ "${VERSION}" == "master" ]]
-              then
-                printf "   Running master... "
-                EXE=$repoPath/build_${JOBNAME}_master/src/MarDyn
-              else
-                printf "   Running new version... "
-                EXE=$repoPath/build_${JOBNAME}/src/MarDyn
-              fi
+            # run the example with the new exe
+            printf "   Running example: $i ... "
+            EXE=$repoPath/build_${JOBNAME}/src/MarDyn
 
-              # when using OpenMPI --oversubscribe is needed. Remove it if you switch to MPICH.
-              mpirun --oversubscribe -np ${{ matrix.procs }} ${EXE} input_patched.xml --steps=20                            \
-                | tee "output_${{ join(matrix.*, '-') }}"                                                   \
-                | awk '/Simstep = /{ print $7 " " $10 " " $13 " " $16 }' > ${repoPath}/output_${VERSION}    \
-                || (exitCode=$?; cat "output_${{ join(matrix.*, '-') }}"; (exit $exitCode))
-              printf "Done\n"
-            done
-            # compare the two runs
-            diff ${repoPath}/output_new ${repoPath}/output_master
+            # when using OpenMPI --oversubscribe is needed. Remove it if you switch to MPICH.
+            mpirun --oversubscribe -np ${{ matrix.procs }} ${EXE} input_patched.xml --steps=20                            \
+              | tee "output_${{ join(matrix.*, '-') }}"                                                   \
+              | awk '/Simstep = /{ print $7 " " $10 " " $13 " " $16 }' > ${repoPath}/output_${VERSION}    \
+              || (exitCode=$?; cat "output_${{ join(matrix.*, '-') }}"; (exit $exitCode))
+            printf "Done\n"
+
+            # compare the output of the run to the validation file
+            # validation files are named after the respective config xml but without the file extension (::-4)
+            python3 ${repoPath}/checks/validation_compare_files.py \
+              --validation-file "validation_$(basename ${i::-4}).json" \
+              --logfile "output_${{ join(matrix.*, '-') }}"
           done
 
       env:

checks/validation_compare_files.py

@@ -0,0 +1,110 @@
+import argparse
+import json
+import numpy as np
+
+from validation_createJSON import parse_resultwriter_file, parse_log_file
+
+def compare_data(new_data, validation_data, reltolerance):
+    '''
+    Compares new and validation data sets using numpy's isclose with a specified tolerance.
+    '''
+    differences = []
+    for i, (new_entry, validation_entry) in enumerate(zip(new_data, validation_data)):
+        entry_differences = {}
+        for key in new_entry.keys():
+            if key in validation_entry:
+                if not np.isclose(new_entry[key], validation_entry[key], rtol=reltolerance):
+                    entry_differences[key] = {
+                        'presentRun': new_entry[key],
+                        'reference': validation_entry[key]
+                    }
+        if entry_differences:
+            if 'Simstep' in new_entry.keys():  # Log file
+                simstep = new_entry['Simstep']
+            elif 'simstep' in new_entry.keys():  # ResultWriter
+                simstep = new_entry['simstep']
+            else:
+                simstep = np.nan
+            differences.append({'index': i, 'simstep': simstep, 'differences': entry_differences})
+    return differences
+
+def compare_validation_file(validation_file, new_log_file):
+    '''
+    Compares the new files with the data stored in the validation JSON file.
+    '''
+    try:
+        with open(validation_file, 'r') as f:
+            validation_data = json.load(f)
+    except FileNotFoundError:
+        print(f'Error: Validation file "{validation_file}" not found.')
+        exit(1)
+    except json.JSONDecodeError:
+        print(f'Error: Validation file "{validation_file}" is not a valid JSON file.')
+        exit(1)
+    except Exception as e:
+        print(f'Failed with exception: {e}')
+        raise
+
+    # Relative tolerance; chosen so that small deviations due to number of ranks are neglected
+    # Specified in metadata
+    reltolerance = validation_data['metadata']['reltolerance']
+
+    # Parse files and compare data; errors are handled in respective function
+
+    # Process log file
+    try:
+        validation_log_data = validation_data['logfile']
+    except KeyError:
+        print('Error: Validation file is missing required key ("logfile").')
+        exit(1)
+    new_log_data = parse_log_file(new_log_file)
+    log_diffs = compare_data(new_log_data, validation_log_data, reltolerance)
+
+    # Process file of ResultWriter
+    try:
+        validation_result_data = validation_data['ResultWriter']
+    except KeyError:
+        print('Error: Validation file is missing required keys ("ResultWriter").')
+        exit(1)
+
+    # Get filename of output of ResultWriter from metadata
+    # Gives "None" if no ResultWriter file was specified during generation of validation file
+    new_result_file = validation_data['metadata']['ResultWriter_filename']
+
+    if new_result_file == "None":
+        new_result_data = []
+    else:
+        new_result_data = parse_resultwriter_file(new_result_file)
+
+    resultwriter_diffs = compare_data(new_result_data, validation_result_data, reltolerance)
+
+    return {
+        'log_diffs': log_diffs,
+        'resultwriter_diffs': resultwriter_diffs
+    }
+
+if __name__ == '__main__':
+    '''
+    Compares the output (Logger, ResultWriter) of a simulation using numpy's isclose function.
+    Since ResultWriter writes to file but Logger to stdout, the logfile has to be specified.
+    '''
+    parser = argparse.ArgumentParser(
+        description='Compare new simulation with a JSON validation file',
+        epilog='Example usage: validation_compare_files.py --validation-file=validation.json --logfile=new_log.log --resultfile=new_result.res',
+    )
+    parser.add_argument('--validation-file', required=True, help='Path to the JSON validation file')
+    parser.add_argument('--logfile', required=True, help='Path to the new log file')
+
+    args = parser.parse_args()
+
+    differences = compare_validation_file(args.validation_file, args.logfile)
+
+    # Print differences
+    if differences['log_diffs'] or differences['resultwriter_diffs']:
+        print('Differences found:')
+        print(json.dumps(differences, indent=4))
+        exit(1)  # Exit with failure
+    else:
+        print('No differences found :-)')
+        exit(0)  # Exit with success
+

checks/validation_createJSON.py

@@ -0,0 +1,113 @@
+import argparse
+import json
+import re
+import os
+from datetime import datetime
+import pandas as pd
+
+def parse_log_file(filepath):
+    '''
+    Parses the output of the Logger.
+
+    Args:
+        filepath (str): Path to the log file.
+
+    Returns:
+        list[dict]: A list of dictionaries representing the parsed data.
+    '''
+    try:
+        data = []
+        # '\d.eE+-' to also match scientific notation and positive/negative numbers
+        pattern = re.compile(r'Simstep = (\d+)\s+T = ([\d.eE+-]+)\s+U_pot = ([\d.eE+-]+)\s+p = ([\d.eE+-]+)')
+
+        with open(filepath, 'r') as f:
+            for line in f:
+                match = pattern.search(line)
+                if match:
+                    data.append({
+                        'Simstep': int(match.group(1)),
+                        'T': float(match.group(2)),
+                        'U_pot': float(match.group(3)),
+                        'p': float(match.group(4))
+                    })
+        return data
+    except Exception as e:
+        raise ValueError(f'Failed to parse log file: {filepath}. Error: {e}')
+
+def parse_resultwriter_file(filepath):
+    '''
+    Parses the file from the ResultWriter using pandas to handle variable columns
+
+    Args:
+        filepath (str): Path to the result file.
+
+    Returns:
+        list[dict]: A list of dictionaries representing the parsed data.
+    '''
+    try:
+        # Read the file, skipping comments and ignoring lines starting with '#'
+        df = pd.read_csv(filepath, delim_whitespace=True, comment='#', engine='python')
+        return df.to_dict(orient='records')
+    except Exception as e:
+        raise ValueError(f'Failed to parse file from ResultWriter: {filepath}. Error: {e}')
+
+def create_validation_file(log_data, result_data, output_file):
+    '''
+    Combines the data into a single JSON file with metadata.
+
+    Args:
+        result_data (dict with name(string) and data(list[dict])): Parsed data from ResultWriter
+        log_data (list[dict]): Parsed data from output of Logger
+        output_file (str): Path to the output JSON file.
+    '''
+
+    output = os.popen('git rev-parse --short HEAD; echo $?').read().split()
+    exit_code = int(output[-1])
+    if exit_code == 0:
+        commit_hash = output[0]
+    else:
+        print('Warning! Commit hash could not be determined!')
+        commit_hash = ''
+
+    validation_data = {
+        'metadata': {
+            'created_at': datetime.now().isoformat(),
+            'commit_at_creation': commit_hash,
+            'build_options': 'CC=gcc CXX=g++ cmake -DVECTOR_INSTRUCTIONS=AVX2 -DCMAKE_BUILD_TYPE=Release -DENABLE_AUTOPAS=OFF -DAUTOPAS_ENABLE_RULES_BASED_AND_FUZZY_TUNING=ON -DENABLE_ALLLBL=OFF -DOPENMP=ON -DENABLE_MPI=ON -DENABLE_UNIT_TESTS=ON -DENABLE_VTK=ON ..',
+            'comment': '',
+            'reltolerance': 1e-8,  # Relative tolerance used for comparison
+            'ResultWriter_filename': result_data['name'],
+        },
+        'logfile' : log_data,
+        'ResultWriter' : result_data['data'],
+    }
+
+    with open(output_file, 'w') as f:
+        json.dump(validation_data, f, indent=4)
+
+    print(f'Validation file created: {output_file}')
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser(
+        description='Process ls1 output files and create a JSON file used for validation',
+        epilog='Example usage: validation_createJSON.py --logfile=out.log --resultfile=result.res --output=validation.json'
+    )
+    parser.add_argument('--logfile', required=True, help='Path to the log file')
+    parser.add_argument('--resultfile', help='Path to the file created by the ResultWriter')
+    parser.add_argument('--output', default='validation.json', help='Path to the output JSON validation file')
+
+    args = parser.parse_args()
+
+    result_data = dict()
+
+    # Parse the input files
+    if args.resultfile is not None:
+        result_data['data'] = parse_resultwriter_file(args.resultfile)
+        result_data['name'] = os.path.basename(args.resultfile)
+    else:
+        result_data['data'] = list(dict())
+        result_data['name'] = 'None'
+    log_data = parse_log_file(args.logfile)
+
+    # Create the validation file
+    create_validation_file(log_data, result_data, args.output)