From f5ac28e9ec5234397b1836ccd36e087b7bfc0a2f Mon Sep 17 00:00:00 2001 From: Shalin Shah Date: Tue, 12 Jun 2018 12:21:45 -0400 Subject: [PATCH] Added post-processing which works with repeatedTasks --- ...rocessedSedMLSimulationResultsWrapper.java | 88 ++++++ .../simulator/sedml/ProcessSedMLResults.java | 296 ++++++++++++++++++ .../sedml/SedMLSBMLSimulatorExecutor.java | 113 ++----- 3 files changed, 405 insertions(+), 92 deletions(-) create mode 100644 src/main/java/org/simulator/sedml/IProcessedSedMLSimulationResultsWrapper.java create mode 100644 src/main/java/org/simulator/sedml/ProcessSedMLResults.java diff --git a/src/main/java/org/simulator/sedml/IProcessedSedMLSimulationResultsWrapper.java b/src/main/java/org/simulator/sedml/IProcessedSedMLSimulationResultsWrapper.java new file mode 100644 index 00000000..91820553 --- /dev/null +++ b/src/main/java/org/simulator/sedml/IProcessedSedMLSimulationResultsWrapper.java @@ -0,0 +1,88 @@ +package org.simulator.sedml; + +import org.jlibsedml.execution.IProcessedSedMLSimulationResults; + +/** + * Non-API Implementation of {@link IProcessedSedMLSimulationResults} + * Because this class is non-API we don't have exhaustive arg checking etc., + * Borrowed from jlibsedml + * @author Shalin Shah + * + */ + class IProcessedSedMLSimulationResultsWrapper implements IProcessedSedMLSimulationResults { + + private double [][] _data; + private String [] _headers; + + IProcessedSedMLSimulationResultsWrapper(double [][]data, String []headers) { + _headers = new String[headers.length]; + System.arraycopy(headers, 0, _headers, 0, headers.length); + _data = new double[data.length][]; + copyDataFromTo(data, _data); + + } + + public String[] getColumnHeaders() { + String[] rc = new String[_headers.length]; + System.arraycopy(_headers, 0, rc, 0, _headers.length); + return rc; + } + + public double[][] getData() { + double[][] copy = new double[_data.length][]; + copyDataFromTo(_data, copy); + return copy; + + } + + private void copyDataFromTo(double[][] data2, double[][] copy) { + int i = 0; + for (double[] row : data2) { + double[] copyRow = new double[row.length]; + System.arraycopy(row, 0, copyRow, 0, row.length); + + copy[i++] = copyRow; + } + + } + + + public int getNumColumns() { + return _headers.length; + } + + public int getNumDataRows() { + return _data.length; + } + + public Double [] getDataByColumnId(String colID) { + int colInd = getIndexByColumnID(colID); + if(colInd == -1){ + return null; + } + Double [] rc = new Double[_data.length]; + for (int i=0; i< _data.length;i++){ + rc[i]=_data[i][colInd]; + } + return rc; + } + + public int getIndexByColumnID(String colID){ + int colInd=-1; + for (int i =0; i< _headers.length;i++){ + if(_headers[i].equals(colID)){ + colInd=i; + } + } + return colInd; + } + + public Double[] getDataByColumnIndex(int index) { + Double [] rc = new Double[_data.length]; + for (int i=0; i< _data.length;i++){ + rc[i]=_data[i][index]; + } + return rc; + } + +} diff --git a/src/main/java/org/simulator/sedml/ProcessSedMLResults.java b/src/main/java/org/simulator/sedml/ProcessSedMLResults.java new file mode 100644 index 00000000..066e13d3 --- /dev/null +++ b/src/main/java/org/simulator/sedml/ProcessSedMLResults.java @@ -0,0 +1,296 @@ +package org.simulator.sedml; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import org.jlibsedml.AbstractTask; +import org.jlibsedml.DataGenerator; +import org.jlibsedml.Output; +import org.jlibsedml.Parameter; +import org.jlibsedml.SedML; +import org.jlibsedml.Variable; +import org.jlibsedml.VariableSymbol; +import org.jlibsedml.execution.IModel2DataMappings; +import org.jlibsedml.execution.IProcessedSedMLSimulationResults; +import org.jlibsedml.execution.IRawSedmlSimulationResults; +import org.jlibsedml.execution.IXPathToVariableIDResolver; +import org.jlibsedml.modelsupport.SBMLSupport; +import org.jmathml.ASTCi; +import org.jmathml.ASTNode; +import org.jmathml.ASTNumber; +import org.jmathml.EvaluationContext; + +import de.binfalse.bflog.LOGGER; + + +/** + * Processes raw simulation results according to instructions specified in the + * {@link DataGenerator} elements specified in the output.
+ * This class is used to process results using information in dataGenerator elements. + * It is similar to jlibsedml's ProcessSedMLResults2 with the added support for working + * with repeatedTasks. + * @author Shalin Shah + * @since 1.5 + */ +public class ProcessSedMLResults { + private Output wanted; + private SedML sedml; + IProcessedSedMLSimulationResults prRes; + + public ProcessSedMLResults(SedML sedml, Output output) { + // Check for nulls + if (sedml == null || output == null) { + throw new IllegalArgumentException(); + } + this.sedml = sedml; + this.wanted = output; + + // Check that required output exists in sedml + boolean found = false; + for (Output o : sedml.getOutputs()) { + if (o.getId().equals(wanted.getId())) { + found = true; + } + } + if (!found) { + throw new IllegalArgumentException("Output [" + wanted.getId() + + "] does not belong the SED-ML object. "); + } + } + + /** + * This method modifies jlibsedml's process method to support dataGenerators for + * repeatedTasks. Processed results can be extracted using getProcessedResult(). + * @param Map> + */ + public void process(Map> res){ + + // Check for nulls + if (res == null) { + throw new IllegalArgumentException(); + } + if(wanted.getAllDataGeneratorReferences().isEmpty()) { + LOGGER.warn("Data generator list is empty!"); + throw new NullPointerException(); + } + + // calculate total number of rows in all the results + int numRows = 0; + for (AbstractTask t : res.keySet()) { + List result = res.get(t); + for(IRawSedmlSimulationResults curRes: result) { + numRows += curRes.getNumDataRows(); + } + } + Map rawTask2Results = new HashMap(); + + // Iterate over all the data generators to process results + List processed = new ArrayList(); + IXPathToVariableIDResolver variable2IDResolver = new SBMLSupport(); + for (String dgId : wanted.getAllDataGeneratorReferences()) { + double[] mutated = new double[numRows]; + processed.add(mutated); + DataGenerator dg = sedml.getDataGeneratorWithId(dgId); + if (dg == null) { + LOGGER.warn("Empty data generator recevied. Correct SED-ML!"); + return; + } + + List vars = dg.getListOfVariables(); + List params = dg.getListOfParameters(); + Map Var2Model = new HashMap(); + Map var2Result = new HashMap(); + Map var2Data = new HashMap(); + String timeID = ""; + // map varIds to result, based upon task reference + for (Variable variable : vars) { + String modelID; + + if (variable.isVariable()) { + // get the task from which this result variable was generated. + modelID = variable2IDResolver.getIdFromXPathIdentifer(variable.getTarget()); + String taskRef = variable.getReference(); + AbstractTask t = sedml.getTaskWithId(taskRef); + + // get results list for this task. If it is repeatedTask then multiple results + List resList = res.get(t); + + // set up lookups to results, raw data and model ID + if (resList.size() > 1) { + // It's a repeatedTask so process each iteration + int index = 0; + for(IRawSedmlSimulationResults curRes: resList) { + + // Add _index to the id of variable for one iteration of RepeatedTask + var2Result.put(variable.getId() + "_" + Integer.toString(index), curRes); + var2Data.put(variable.getId() + "_" + Integer.toString(index), rawTask2Results.get(t)); + Var2Model.put(variable.getId() + "_" + Integer.toString(index), modelID); + index++; + } + }else { + // Just a Task so get first element + var2Result.put(variable.getId(), resList.get(0)); + var2Data.put(variable.getId(), rawTask2Results.get(t)); + Var2Model.put(variable.getId(), modelID); + } + + // it's a symbol + } else if (variable.isSymbol() + && variable.getSymbol().equals(VariableSymbol.TIME)) { + timeID = variable.getId(); + var2Data.put(variable.getId(), rawTask2Results.values().iterator() + .next()); + Var2Model.put(variable.getId(), variable.getId()); + + } + } + + // get Parameter values + Map Param2Value = new HashMap(); + for (Parameter p : params) { + Param2Value.put(p.getId(), p.getValue()); + } + + // now parse maths, and replace raw simulation results with + // processed results. + ASTNode node = dg.getMath(); + Set identifiers = node.getIdentifiers(); + for (ASTCi var : identifiers) { + if (var.isVector()) { + String varName = var.getName(); + IModel2DataMappings coll = var2Result.get(varName).getMappings(); + int otherVarInx = coll.getColumnIndexFor(Var2Model.get(varName)); + if (otherVarInx < 0 || otherVarInx >= var2Result.get(varName).getNumColumns()) { + LOGGER.warn("No data column for " + var); + return; + } + EvaluationContext con = new EvaluationContext(); + Double[] data = var2Result.get(varName).getDataByColumnIndex(otherVarInx); + + con.setValueFor(varName, Arrays.asList(data)); + + if (var.getParentNode() == null || var.getParentNode().getParentNode() == null) { + LOGGER.warn("Could not evaluate [" + var + "] as symbol does not have parent element"); + return; + } + if (!var.getParentNode().canEvaluate(con)) { + LOGGER.warn("Could not evaluate [" + var + "]"); + return; + } + ASTNumber num = var.getParentNode().evaluate(con); + // replace vector operation with calculated value. + var.getParentNode().getParentNode().replaceChild(var.getParentNode(), num); + } + } + // identifiers.add(var.getSpId()); + if (identifiersMapToData(identifiers, Var2Model, Param2Value, var2Result, timeID)) { + + for (int i = 0; i < numRows; i++) { + EvaluationContext con = new EvaluationContext(); + + for (String id : Param2Value.keySet()) { + con.setValueFor(id, Param2Value.get(id)); + } + + for (ASTCi var : identifiers) { + // we've already resolved parameters + if (Param2Value.get(var.getName()) != null) { + continue; + } + int otherVarInx = 0; + if (!var.getName().equals(timeID)) { + IModel2DataMappings coll = var2Result.get( + var.getName()).getMappings(); + otherVarInx = coll.getColumnIndexFor(Var2Model + .get(var.getName())); + if (otherVarInx < 0 + || otherVarInx >= var2Result.get( + var.getName()).getNumColumns()) { + LOGGER.warn("No data column for " + var); + return; + } + } + con.setValueFor(var.getName(), + var2Data.get(var.getName())[i][otherVarInx]); + } + + if (node.canEvaluate(con)) { + mutated[i] = node.evaluate(con).getValue(); + } else { + LOGGER.warn("Math could not be executed for data generator " + dgId); + } + } + } else { + LOGGER.warn("Math could not be executed for data generator " + dgId); + return; + } + } + + prRes = createData(processed, numRows); + } + + // Helper method for processing simulation results as per dataGenerator instructions + // Borrowed from jlibsedml library to deal with repeatedTasks + private boolean identifiersMapToData(Set identifiers, + Map Var2Model, Map Param2Value, + Map var2Result, String timeID) { + + for (ASTCi var : identifiers) { + boolean seen = false; + if (Param2Value.get(var.getName()) != null) { + seen = true; + } else if (Var2Model.get(var.getName()) != null) { + if (var.getName().equals(timeID)) { + seen = true; + } else { + IModel2DataMappings coll = var2Result.get(var.getName()) + .getMappings(); + if (coll.hasMappingFor(Var2Model.get(var.getName())) + && coll.getColumnTitleFor(Var2Model.get(var + .getName())) != null + || var.getName().equals(timeID)) { + seen = true; + } + } + } + + if (!seen) { + return false; + } + + } + return true; + } + + // Helper method for processing simulation results as per dataGenerator instructions + // Borrowed from jlibsedml library to deal with repeatedTasks + private IProcessedSedMLSimulationResults createData( + List processed, int NumRows) { + + String[] hdrs = new String[processed.size()]; + int colInd = 0; + for (Iterator it = wanted.getAllDataGeneratorReferences() + .iterator(); it.hasNext();) { + hdrs[colInd++] = it.next(); + } + + double[][] data = new double[NumRows][hdrs.length]; + for (int j = 0; j < NumRows; j++) { + for (int i = 0; i < hdrs.length; i++) { + data[j][i] = processed.get(i)[j]; + } + + } + return new IProcessedSedMLSimulationResultsWrapper(data, hdrs); + + } + + public IProcessedSedMLSimulationResults getProcessedResult() { + return prRes; + } +} diff --git a/src/main/java/org/simulator/sedml/SedMLSBMLSimulatorExecutor.java b/src/main/java/org/simulator/sedml/SedMLSBMLSimulatorExecutor.java index 0e54b884..19300856 100644 --- a/src/main/java/org/simulator/sedml/SedMLSBMLSimulatorExecutor.java +++ b/src/main/java/org/simulator/sedml/SedMLSBMLSimulatorExecutor.java @@ -28,20 +28,17 @@ import java.io.File; import java.util.ArrayList; import java.util.Arrays; -import java.util.Collection; import java.util.Collections; import java.util.Comparator; import java.util.HashMap; +import java.util.Iterator; import java.util.LinkedHashMap; import java.util.LinkedList; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.Map.Entry; -import java.util.stream.Collectors; -import java.util.stream.Stream; - import org.apache.commons.io.FileUtils; -import org.apache.commons.lang.ArrayUtils; import org.apache.log4j.Logger; import org.jlibsedml.AbstractTask; import org.jlibsedml.ArchiveComponents; @@ -51,7 +48,6 @@ import org.jlibsedml.Parameter; import org.jlibsedml.Range; import org.jlibsedml.RepeatedTask; -import org.jlibsedml.SEDMLVisitor; import org.jlibsedml.SedML; import org.jlibsedml.SetValue; import org.jlibsedml.Simulation; @@ -63,22 +59,22 @@ import org.jlibsedml.VariableSymbol; import org.jlibsedml.execution.AbstractSedmlExecutor; import org.jlibsedml.execution.ArchiveModelResolver; -import org.jlibsedml.execution.ExecutionStatusElement; import org.jlibsedml.execution.FileModelResolver; import org.jlibsedml.execution.IModel2DataMappings; -import org.jlibsedml.execution.ExecutionStatusElement.ExecutionStatusType; import org.jlibsedml.execution.IProcessedSedMLSimulationResults; import org.jlibsedml.execution.IRawSedmlSimulationResults; import org.jlibsedml.execution.IXPathToVariableIDResolver; import org.jlibsedml.execution.ModelResolver; -import org.jlibsedml.execution.SedMLResultsProcesser2; import org.jlibsedml.modelsupport.BioModelsModelsRetriever; import org.jlibsedml.modelsupport.KisaoOntology; import org.jlibsedml.modelsupport.KisaoTerm; import org.jlibsedml.modelsupport.SBMLSupport; import org.jlibsedml.modelsupport.URLResourceRetriever; +import org.jmathml.ASTCi; +import org.jmathml.ASTNode; +import org.jmathml.ASTNumber; +import org.jmathml.EvaluationContext; import org.sbml.jsbml.Model; -import org.sbml.jsbml.validator.offline.constraints.ArraysUtils; import org.sbml.jsbml.xml.stax.SBMLReader; import org.simulator.math.odes.AbstractDESSolver; import org.simulator.math.odes.DormandPrince54Solver; @@ -369,8 +365,8 @@ public Map> run() { return res; } if (sim == null || !canExecuteSimulation(sim)) { - LOGGER.warn("Cannot simulate task" + relatedTask.getId() - + "Either the simulation reference is corrupt or the simulation algorithm is not available."); + LOGGER.warn("Cannot simulate task " + relatedTask.getId() + + " Either the simulation reference is corrupt or the simulation algorithm is not available."); return res; } @@ -428,8 +424,8 @@ public Map> run() { return res; } if (sim == null || !canExecuteSimulation(sim)) { - LOGGER.warn("Cannot simulate task" + stdTask.getId() - + "Either the simulation reference is corrupt or the simulation algorithm is not available."); + LOGGER.warn("Cannot simulate task " + stdTask.getId() + + " Either the simulation reference is corrupt or the simulation algorithm is not available."); return res; } if (changedModel == null) { @@ -456,7 +452,6 @@ public Map> run() { } return res; - } /** @@ -560,83 +555,18 @@ AbstractDESSolver getSolverForKisaoID(String id) { public MultiTable processSimulationResults(Output wanted, Map> res) { -// // Check for nulls -// if (sedml == null || wanted == null || res == null) { -// throw new IllegalArgumentException(); -// } -// if(wanted.getAllDataGeneratorReferences().isEmpty()) { -// LOGGER.warn("Data generator list is empty!"); -// throw new NullPointerException(); -// } -// // Check that required output exisits in sedml -// boolean found = false; -// for (Output o : sedml.getOutputs()) { -// if (o.getId().equals(wanted.getId())) { -// found = true; -// } -// } -// if (!found) { -// throw new IllegalArgumentException("Output [" + wanted.getId() -// + "] does not belong the SED-ML object. "); -// } -// // calculate total number of rows in all the results -// int numRows = 0; -// for (AbstractTask t : res.keySet()) { -// List result = res.get(t); -// for(IRawSedmlSimulationResults curRes: result) { -// numRows += curRes.getNumDataRows(); -// } -// } -// -// // Iterate over all the data generators and to process results -// List processed = new ArrayList(); -// IXPathToVariableIDResolver variable2IDResolver = new SBMLSupport(); -// for (String dgId : wanted.getAllDataGeneratorReferences()) { -// double[] mutated = new double[numRows]; -// processed.add(mutated); -// DataGenerator dg = sedml.getDataGeneratorWithId(dgId); -// if (dg == null) { -// LOGGER.warn("Empty data generator recevied. Correct SED-ML!"); -// return null; -// } -// -// List vars = dg.getListOfVariables(); -// List params = dg.getListOfParameters(); -// Map Var2Model = new HashMap(); -// Map var2Result = new HashMap(); -// Map var2Data = new HashMap(); -// String timeID = ""; -// // map varIds to result, based upon task reference -// for (Variable variable : vars) { -// String modelID; -// -// if (variable.isVariable()) { -// // get the task from which this result variable was generated. -// modelID = variable2IDResolver.getIdFromXPathIdentifer(variable.getTarget()); -// String taskRef = variable.getReference(); -// AbstractTask t = sedml.getTaskWithId(taskRef); -// -// // get results list for this task. If it is repeatedTask then multiple results -// List resList = res.get(t); -// // set up lookups to results, raw data and model ID -// var2Result.put(variable.getId(), res); -// var2Data.put(variable.getId(), rawTask2Results.get(t)); -// Var2Model.put(variable.getId(), modelID); -// // it's a symbol -// } else if (variable.isSymbol() -// && variable.getSymbol().equals(VariableSymbol.TIME)) { -// timeID = variable.getId(); -// var2Data.put(variable.getId(), rawTask2Results.values().iterator() -// .next()); -// Var2Model.put(variable.getId(), variable.getId()); -// -// } -// } -// -// } + // here we post-process the results using our own post-processing module + ProcessSedMLResults pcsr = new ProcessSedMLResults(sedml, wanted); + pcsr.process(res); - - return null; + // this does not necessarily have time as x-axis - another variable could be the + // independent variable. + IProcessedSedMLSimulationResults prRes = pcsr.getProcessedResult(); + + // now we restore a MultiTable from the processed results. This basic example assumes a typical + // simulation where time = xaxis - otherwise, if output is a Plot, we would need to analyse the x-axis + // datagenerators + return createMultiTableFromProcessedResults(wanted, prRes); } // Here we need to check which of the results are the independent axis to create a MultiTable @@ -722,5 +652,4 @@ private String findTimeColumn(IProcessedSedMLSimulationResults prRes, } return null; } - }