Merge pull request #1 from lambda-feedback/auto-docs-for-tests

Added the ability to write tests in JSON and run them automatically. This is intended to make it easier to auto-generate documentation from the evaluation_function unit tests.

Author: m-hurlow

.DS_Store

@@ -0,0 +1 @@
+test_output.json

docs/.gitignore

@@ -0,0 +1,76 @@
+[
+    {
+        "title": "Basic expression equality and inequality",
+        "masterContent": "Demonstrates trivial comparisons",
+        "parts": [
+            {
+                "content": "The response and answer are exactly the same, so the response should be considered correct.",
+                "responseAreas": [
+                    {
+                        "preResponseText": "",
+                        "answer": "A & B",
+                        "params": {},
+                        "tests": [
+                            {
+                                "description": "Most basic possible case",
+                                "response": "A & B",
+                                "expectedResult": {
+                                    "is_correct": true,
+                                    "response_latex": "A \\cdot B"
+                                }
+                            }
+                        ]
+                    },
+                    {
+                        "preResponseText": "Multi-character variable names are supported.",
+                        "answer": "A & Test",
+                        "params": {},
+                        "tests": [
+                            {
+                                "description": "Works with variable names of any length",
+                                "response": "A & Test",
+                                "expectedResult": {
+                                    "is_correct": true,
+                                    "response_latex": "A \\cdot \\mathrm{Test}"
+                                }
+                            }
+                        ]
+                    }
+                ]
+            },
+            {
+                "content": "",
+                "responseAreas": [
+                    {
+                        "preResponseText": "Transposition of variables:",
+                        "answer": "A & B",
+                        "params": {},
+                        "tests": [
+                            {
+                                "description": "Tests transposed variables are correct",
+                                "response": "B & A",
+                                "expectedResult": {
+                                    "is_correct": true
+                                }
+                            }
+                        ]
+                    },
+                    {
+                        "preResponseText": "Trivially incorrect response:",
+                        "answer": "A & B",
+                        "params": {},
+                        "tests": [
+                            {
+                                "description": "Incorrect results marked as false",
+                                "response": "A | B",
+                                "expectedResult": {
+                                    "is_correct": false
+                                }
+                            }
+                        ]
+                    }
+                ]
+            }
+        ]
+    }
+]

eval_tests.json

@@ -14,7 +14,7 @@ def evaluation_function(
     response: Any,
     answer: Any,
     params: Params,
-) -> dict:
+) -> Result:
     """
     Function used to evaluate a student response.
     ---

evaluation_function/evaluation.py

@@ -1,7 +1,9 @@
 import unittest
 
 from .evaluation import Params, evaluation_function
+from .json_tests import auto_test
 
+@auto_test("eval_tests.json", evaluation_function)
 class TestEvaluationFunction(unittest.TestCase):
     """
     TestCase Class used to test the algorithm.
@@ -21,15 +23,18 @@ class TestEvaluationFunction(unittest.TestCase):
     as it should.
     """
 
+    # Tests
+
     def test_returns_is_correct_true_ascii(self):
         response, answer, params = "A & Test", "A & Test", Params()
 
         result = evaluation_function(response, answer, params).to_dict()
 
         self.assertEqual(result.get("is_correct"), True)
-        self.assertEqual(result.get("response_latex"), "A \\cdot \mathrm{Test}")
+        self.assertEqual(result.get("response_latex"),
+                         "A \\cdot \mathrm{Test}")
         self.assertFalse(result.get("feedback"))
-    
+
     def test_returns_negative(self):
         response, answer, params = "A | B", "A & B", Params()
 
@@ -38,7 +43,7 @@ def test_returns_negative(self):
         self.assertEqual(result.get("is_correct"), False)
         self.assertEqual(result.get("response_latex"), "A + B")
         self.assertTrue(result.get("feedback"))
-    
+
     def test_syntax_error(self):
         response, answer, params = "A £ B", "A & B", Params()
 
@@ -47,7 +52,7 @@ def test_syntax_error(self):
             self.fail("Exception not raised for unexpected token")
         except:
             pass
-    
+
     def test_xor_identity(self):
         response, answer, params = "A ^ B", "A & ~B | ~A & B", Params()
 
@@ -56,7 +61,7 @@ def test_xor_identity(self):
         self.assertEqual(result.get("is_correct"), True)
         self.assertEqual(result.get("response_latex"), "A \\oplus B")
         self.assertFalse(result.get("feedback"))
-    
+
     def test_nand_or(self):
         response, answer, params = "A | B", "~(~A & ~B)", Params()
 
@@ -65,7 +70,7 @@ def test_nand_or(self):
         self.assertEqual(result.get("is_correct"), True)
         self.assertEqual(result.get("response_latex"), "A + B")
         self.assertFalse(result.get("feedback"))
-    
+
     def test_nand_or(self):
         response, answer, params = "A | B", "~(~A & ~B)", Params()
 
@@ -81,9 +86,10 @@ def test_nor_nand(self):
         result = evaluation_function(response, answer, params).to_dict()
 
         self.assertEqual(result.get("is_correct"), True)
-        self.assertEqual(result.get("response_latex"), "\\overline{\\left( A \\cdot B \\right)}")
+        self.assertEqual(result.get("response_latex"),
+                         "\\overline{\\left( A \\cdot B \\right)}")
         self.assertFalse(result.get("feedback"))
-    
+
     def test_complex(self):
         response, answer, params = "A & B | B & C & (B | C)", "B & (A | C)", Params()
 

evaluation_function/evaluation_test.py

@@ -0,0 +1,71 @@
+import json
+
+class TestData:
+    def __init__(self, test_dict: dict):
+        self.response = test_dict["response"]
+        self.answer = test_dict["answer"]
+        self.params = test_dict["params"]
+        expected_result = test_dict["expectedResult"]
+        self.is_correct = expected_result["is_correct"]
+        self.results = expected_result
+        self.desc = test_dict["description"]
+
+    def evaluate(self, func) -> dict:
+        return func(self.response, self.answer, self.params)
+    
+    def compare(self, eval_result: dict) -> tuple[bool, str]:
+        eval_correct = eval_result["is_correct"]
+            
+        if eval_correct != self.is_correct:
+            return (
+                False,
+                f"response \"{self.response}\" with answer \"{self.answer}\" was {'' if eval_correct else 'in'}correct: {eval_result['feedback']}\nTest description: {self.desc}"
+            )
+        
+        # Are there any other fields in the eval function result that need to be checked?
+        if self.results != None:
+            # Check each one in turn
+            for key, value in self.results.items():
+                actual_result_val = eval_result.get(key)
+                if actual_result_val == None:
+                    return (False, f"No value returned for \"{key}\"")
+                
+                if actual_result_val != value:
+                    return (
+                        False,
+                        f"expected {key} = \"{value}\", got {key} = \"{actual_result_val}\"\nTest description: {self.desc}"
+                    )
+        
+        return (True, "")
+        
+
+def get_tests_from_json(filename: str) -> list[TestData]:
+    out = []
+    questions = []
+    with open(filename, "r") as test_file:
+        test_json = test_file.read()
+        questions = json.loads(test_json)
+    # Convert the structured test data into a flat list of tests
+    for question in questions:
+        for part in question["parts"]:
+            for response_area in part["responseAreas"]:
+                params = response_area["params"]
+                answer = response_area["answer"]
+                for test in response_area["tests"]:
+                    test.update({"answer": answer})
+                    test.update({"params": params})
+                    out.append(TestData(test))
+    
+    return out
+
+def auto_test(path, func):
+    def _auto_test(orig_class):
+        def test_auto(self):
+            tests = get_tests_from_json(path)
+            for test in tests:
+                results = test.evaluate(func)
+                self.assertTrue(*test.compare(results.to_dict()))
+
+        orig_class.test_auto = test_auto # Add the test_auto function to the class
+        return orig_class
+    return _auto_test