Merge pull request #154 from lambda-feedback/tr129-experiment-with-syntactical-equivalence

Tr129 experiment with syntactical equivalence

Author: KarlLundengaard

app/evaluation_response_utilities.py

@@ -36,7 +36,13 @@ def add_criteria_graph(self, name, graph):
         self._criteria_graphs.update({name: graph.json()})
 
     def _serialise_feedback(self) -> str:
-        return "<br>".join(x[1] if (isinstance(x, tuple) and len(x[1].strip())) > 0 else x for x in self._feedback)
+        feedback = []
+        for x in self._feedback:
+            if (isinstance(x, tuple) and len(x[1].strip())) > 0:
+                feedback.append(x[1].strip())
+            elif len(x.strip()) > 0:
+                feedback.append(x.strip())
+        return "<br>".join(feedback)
 
     def serialise(self, include_test_data=False) -> dict:
         out = dict(is_correct=self.is_correct, feedback=self._serialise_feedback())

app/feedback/symbolic_comparison.py

@@ -1,5 +1,6 @@
 from ..criteria_utilities import Criterion
 
+
 # TODO: Find better way of identifying reference criteria
 # equivalences dictionary should contain a list of variations that are likely to be produced by the following procedure:
 # - rewrite critera as expr=0,
@@ -8,18 +9,13 @@
 equivalences = dict()
 criteria = dict()
 
-criteria["RESPONSE_EQUAL_ANSWER"] = Criterion("response=answer")
-equivalences.update({"RESPONSE_EQUAL_ANSWER": ["response=answer", "answer=response", "answer-response=0", "-answer+response=0", "answer/response=1", "response/answer-1=0"]})
-criteria["RESPONSE_EQUAL_ANSWER"][True] = lambda inputs: "The response matches the expected answer."
-criteria["RESPONSE_EQUAL_ANSWER"][False] = lambda inputs: "The response does not match the expected answer."
-
 criteria["RESPONSE_DOUBLE_ANSWER"] = Criterion("response=2*answer")
-equivalences.update({"RESPONSE_DOUBLE_ANSWER": ["response=2*answer","response/answer=2", "2*answer=response", "answer=response/2", "answer-response/2", "-answer+response/2", "-2*answer+response", "2*answer-response", "-2+answer/response", "-2+response/answer", "answer-1*response/2", "-answer+1*response/2", "-2+1*answer/response", "-2+1*response/answer"]})
+equivalences.update({"RESPONSE_DOUBLE_ANSWER": ["response=2*answer", "response/answer=2", "2*answer=response", "answer=response/2", "answer-response/2", "-answer+response/2", "-2*answer+response", "2*answer-response", "-2+answer/response", "-2+response/answer", "answer-1*response/2", "-answer+1*response/2", "-2+1*answer/response", "-2+1*response/answer"]})
 criteria["RESPONSE_DOUBLE_ANSWER"][True] = lambda inputs: "The response is the expected answer multiplied by 2."
 criteria["RESPONSE_DOUBLE_ANSWER"][False] = lambda inputs: "The response is not the expected answer multiplied by 2."
 
 criteria["RESPONSE_NEGATIVE_ANSWER"] = Criterion("response=-answer")
-equivalences.update({"RESPONSE_NEGATIVE_ANSWER": ["response=-answer", "answer=-response", "answer+response=0", "answer+response","answer/response=-1", "response/answer+1"]})
+equivalences.update({"RESPONSE_NEGATIVE_ANSWER": ["response=-answer", "answer=-response", "answer+response=0", "answer+response", "answer/response=-1", "response/answer+1"]})
 criteria["RESPONSE_NEGATIVE_ANSWER"][True] = lambda inputs: "The response is the expected answer multiplied by -1."
 criteria["RESPONSE_NEGATIVE_ANSWER"][False] = lambda inputs: "The response is not the expected answer multiplied by -1."
 
@@ -65,7 +61,7 @@
     "EXPRESSION_NOT_EQUALITY": "The response was an expression but was expected to be an equality.",
     "EQUALITY_NOT_EXPRESSION": "The response was an equality but was expected to be an expression.",
     "WITHIN_TOLERANCE": None,  # "The difference between the response the answer is within specified error tolerance.",
-    "NOT_NUMERICAL": None,  #"The expression cannot be evaluated numerically.",
+    "NOT_NUMERICAL": None,  # "The expression cannot be evaluated numerically.",
 }[tag]
 feedback_generators["GENERIC"] = lambda tag: lambda inputs: {
     "TRUE": None,

app/symbolic_comparison_evaluation.py

@@ -228,7 +228,7 @@ def same_symbols(unused_input):
         label+"_SAME_SYMBOLS"+"_TRUE",
         summary=str(lhs)+" has the same symbols as "+str(rhs),
         details=str(lhs)+" has the same (free) symbols as "+str(rhs)+".",
-        feedback_string_generator=symbolic_feedback_generators["response=answer"]("FALSE")
+        feedback_string_generator=symbolic_feedback_generators["SAME_SYMBOLS"]("FALSE")
     )
     graph.attach(label+"_SAME_SYMBOLS"+"_TRUE", END.label)
     graph.attach(

app/symbolic_comparison_evaluation_tests.py

@@ -1052,11 +1052,11 @@ def test_no_reserved_keywords_in_old_format_input_symbol_alternatives(self):
     @pytest.mark.parametrize(
         "response, answer, criteria, value, feedback_tags, additional_params",
         [
-            ("a+b", "b+a", "answer=response", True, ["RESPONSE_EQUAL_ANSWER"], {}),
+            ("a+b", "b+a", "answer=response", True, ["answer=response_TRUE"], {}),
             #("a+b", "b+a", "not(answer=response)", False, [], {}),
-            ("a+b", "b+a", "answer-response=0", True, ["RESPONSE_EQUAL_ANSWER"], {}),
-            ("a+b", "b+a", "answer/response=1", True, ["RESPONSE_EQUAL_ANSWER"], {}),
-            ("a+b", "b+a", "answer=response, answer-response=0, answer/response=1", True, ["RESPONSE_EQUAL_ANSWER"], {}),
+            ("a+b", "b+a", "answer-response=0", True, ["answer-response=0_TRUE"], {}),
+            ("a+b", "b+a", "answer/response=1", True, ["answer/response=1_TRUE"], {}),
+            ("a+b", "b+a", "answer=response, answer-response=0, answer/response=1", True, ["answer=response_TRUE", "answer-response=0_TRUE", "answer/response=1_TRUE"], {}),
             ("2a", "a", "response/answer=2", True, ["RESPONSE_DOUBLE_ANSWER"], {}),
             ("2a", "a", "2*answer = response", True, ["RESPONSE_DOUBLE_ANSWER"], {}),
             ("2a", "a", "answer = response/2", True, ["RESPONSE_DOUBLE_ANSWER"], {}),
@@ -1145,21 +1145,21 @@ def test_disabled_evaluation_nodes(self, response, answer, criteria, value, disa
     @pytest.mark.parametrize(
         "response, answer, criteria, value, feedback_tags, additional_params",
         [
-            ("2", "2", "response=answer", True, ["response=answer_SYNTACTICAL_EQUIVALENCE_TRUE", "response=answer_SAME_SYMBOLS_TRUE"], {}),
-            ("4/2", "2", "answer=response", True, ["answer=response_SYNTACTICAL_EQUIVALENCE_FALSE"], {}),
-            ("2+x-x", "2", "answer=response", True, ["answer=response_SAME_SYMBOLS_FALSE"], {}),
-            ("2+2*I", "2+2*I", "answer=response", True, ["answer=response_TRUE", "answer=response_SAME_FORM_CARTESIAN"], {}),
-            ("2+2I", "2+2*I", "answer=response", True, ["answer=response_TRUE", "answer=response_SAME_FORM_CARTESIAN"], {}),
-            ("2.00+2.00*I", "2+2*I", "answer=response", True, ["answer=response_TRUE", "answer=response_SAME_FORM_CARTESIAN"], {}),
+            ("2", "2", "response=answer", True, ["response=answer_TRUE", "response=answer_SYNTACTICAL_EQUIVALENCE_TRUE", "response=answer_SAME_SYMBOLS_TRUE", "response=answer_SAME_FORM_CARTESIAN"], {}),
+            ("4/2", "2", "answer=response", True, ["answer=response_TRUE", "answer=response_SAME_SYMBOLS_TRUE", "answer=response_SYNTACTICAL_EQUIVALENCE_FALSE", "answer=response_SAME_FORM_UNKNOWN"], {}),
+            ("2+x-x", "2", "answer=response", True, ["answer=response_TRUE", "answer=response_SAME_FORM_UNKNOWN", "answer=response_SYNTACTICAL_EQUIVALENCE_FALSE", "answer=response_SAME_SYMBOLS_FALSE"], {}),
+            ("2+2*I", "2+2*I", "answer=response", True, ["answer=response_TRUE", "answer=response_SAME_SYMBOLS_TRUE", "answer=response_SYNTACTICAL_EQUIVALENCE_TRUE", "answer=response_SAME_FORM_CARTESIAN"], {}),
+            ("2+2I", "2+2*I", "answer=response", True, ["answer=response_TRUE", "answer=response_SAME_SYMBOLS_TRUE", "answer=response_SYNTACTICAL_EQUIVALENCE_FALSE", "answer=response_SAME_FORM_CARTESIAN"], {}),
+            ("2.00+2.00*I", "2+2*I", "answer=response", True, ["answer=response_TRUE", "answer=response_SAME_SYMBOLS_TRUE", "answer=response_SYNTACTICAL_EQUIVALENCE_FALSE", "answer=response_SAME_FORM_CARTESIAN"], {}),
             ("3+3I", "2+2*I", "answer=response", False, ["answer=response_FALSE", "answer=response_SAME_FORM_CARTESIAN"], {}),
-            ("2(1+I)", "2+2*I", "answer=response", True, ["answer=response_TRUE", "answer=response_SAME_FORM_UNKNOWN"], {}),
-            ("2I+2", "2+2*I", "answer=response", True, ["answer=response_TRUE", "answer=response_SAME_FORM_UNKNOWN"], {}),
-            ("4/2+6/3*I", "2+2*I", "answer=response", True, ["answer=response_TRUE", "answer=response_SAME_FORM_UNKNOWN"], {}),
-            ("2*e^(2*I)", "2*e^(2*I)", "answer=response", True, ["answer=response_TRUE", "answer=response_SAME_FORM_EXPONENTIAL"], {}),
-            ("2*E^(2*I)", "2*e^(2*I)", "answer=response", True, ["answer=response_TRUE", "answer=response_SAME_FORM_EXPONENTIAL"], {}),
-            ("2*exp(2*I)", "2*e^(2*I)", "answer=response", True, ["answer=response_TRUE", "answer=response_SAME_FORM_EXPONENTIAL"], {}),
-            ("2*e**(2*I)", "2*e^(2*I)", "answer=response", True, ["answer=response_TRUE", "answer=response_SAME_FORM_EXPONENTIAL"], {}),
-            ("e**(2*I)", "1*e^(2*I)", "answer=response", True, ["answer=response_TRUE", "answer=response_SAME_FORM_EXPONENTIAL"], {}),
+            ("2(1+I)", "2+2*I", "answer=response", True, ["answer=response_TRUE", "answer=response_SAME_SYMBOLS_TRUE", "answer=response_SYNTACTICAL_EQUIVALENCE_FALSE", "answer=response_SAME_FORM_UNKNOWN"], {}),
+            ("2I+2", "2+2*I", "answer=response", True, ["answer=response_TRUE", "answer=response_SAME_SYMBOLS_TRUE", "answer=response_SYNTACTICAL_EQUIVALENCE_FALSE", "answer=response_SAME_FORM_UNKNOWN"], {}),
+            ("4/2+6/3*I", "2+2*I", "answer=response", True, ["answer=response_TRUE", "answer=response_SAME_SYMBOLS_TRUE", "answer=response_SYNTACTICAL_EQUIVALENCE_FALSE", "answer=response_SAME_FORM_UNKNOWN"], {}),
+            ("2*e^(2*I)", "2*e^(2*I)", "answer=response", True, ["answer=response_TRUE", "answer=response_SAME_SYMBOLS_TRUE", "answer=response_SYNTACTICAL_EQUIVALENCE_TRUE", "answer=response_SAME_FORM_EXPONENTIAL"], {}),
+            ("2*E^(2*I)", "2*e^(2*I)", "answer=response", True, ["answer=response_TRUE", "answer=response_SAME_SYMBOLS_TRUE", "answer=response_SYNTACTICAL_EQUIVALENCE_TRUE", "answer=response_SAME_FORM_EXPONENTIAL"], {}),
+            ("2*exp(2*I)", "2*e^(2*I)", "answer=response", True, ["answer=response_TRUE", "answer=response_SAME_SYMBOLS_TRUE", "answer=response_SYNTACTICAL_EQUIVALENCE_FALSE", "answer=response_SAME_FORM_EXPONENTIAL"], {}),
+            ("2*e**(2*I)", "2*e^(2*I)", "answer=response", True, ["answer=response_TRUE", "answer=response_SAME_SYMBOLS_TRUE", "answer=response_SYNTACTICAL_EQUIVALENCE_FALSE", "answer=response_SAME_FORM_EXPONENTIAL"], {}),
+            ("e**(2*I)", "1*e^(2*I)", "answer=response", True, ["answer=response_TRUE", "answer=response_SAME_SYMBOLS_TRUE", "answer=response_SYNTACTICAL_EQUIVALENCE_FALSE", "answer=response_SAME_FORM_EXPONENTIAL"], {}),
             ("0.48+0.88*i", "1*e^(0.5*I)", "answer=response", False, ["answer=response_FALSE", "answer=response_SAME_FORM_UNKNOWN"], {}),
         ]
     )
@@ -1173,8 +1173,7 @@ def test_syntactical_comparison(self, response, answer, criteria, value, feedbac
         params.update(additional_params)
         result = evaluation_function(response, answer, params, include_test_data=True)
         assert result["is_correct"] is value
-        for feedback_tag in feedback_tags:
-            assert feedback_tag in result["tags"]
+        assert set(feedback_tags) == set(result["tags"])
 
     @pytest.mark.parametrize(
         "response, answer, value",