Skip to content

Commit

Permalink
Merge pull request #533 from joelostblom/test-emojis
Browse files Browse the repository at this point in the history
  • Loading branch information
@chrispyles
chrispyles authored Oct 8, 2022
2 parents 7c00b46 + cc66129 commit e281f10
Show file tree
Hide file tree
Showing 7 changed files with 22 additions and 11 deletions.
4 changes: 4 additions & 0 deletions CHANGELOG.md
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,9 @@
# Changelog

**Unreleased:**

* Display emojis with test results to more clearly show which tests pass and which fail, as per [#533](https://github.com/ucbds-infra/otter-grader/pull/533).

**v4.0.2:**

* Close temporary file handle before removal when checking tests in `otter.Notebook.export`
Expand Down
6 changes: 4 additions & 2 deletions otter/test_files/abstract_test.py
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
"""Abstract test objects for providing a schema to write and parse test cases"""

import random
from abc import ABC, abstractmethod
from dataclasses import asdict, dataclass, replace
from textwrap import indent
Expand Down Expand Up @@ -72,13 +73,14 @@ class TestFile(ABC):

def _repr_html_(self):
if self.passed_all:
all_passed_emoji = random.choice(['🍀', '🎉', '🌈', '🙌', '🚀', '🌟', '✨', '💯'])
if any(tcr.test_case.success_message is not None for tcr in self.test_case_results):
ret = f"<p><strong><pre style='display: inline;'>{self.name}</pre></strong> passed!</p>"
ret = f"<p><strong><pre style='display: inline;'>{self.name}</pre></strong> passed! {all_passed_emoji}</p>"
for tcr in self.test_case_results:
if tcr.test_case.success_message is not None:
ret += f"<p><strong><pre style='display: inline;'>{tcr.test_case.name}</pre> message:</strong> {tcr.test_case.success_message}</p>"
return ret
return f"<p><strong><pre style='display: inline;'>{self.name}</pre></strong> passed!</p>"
return f"<p><strong><pre style='display: inline;'>{self.name}</pre></strong> passed! {all_passed_emoji}</p>"
else:
ret = f"<p><strong style='color: red;'><pre style='display: inline;'>{self.name}</pre> results:</strong></p>"
for tcr in self.test_case_results:
Expand Down
4 changes: 2 additions & 2 deletions otter/test_files/exception_test.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -164,11 +164,11 @@ def run(self, global_environment):
test_case_results = []
for tc in self.test_cases:
test_case = tc.body
passed, message = True, "Test case passed!"
passed, message = True, "✅ Test case passed"
try:
test_case.call_func(global_environment)
except Exception as e:
passed, message = False, self._generate_error_message(e)
passed, message = False, "❌ Test case failed\n" + self._generate_error_message(e)

test_case_results.append(TestCaseResult(test_case=tc, message=message, passed=passed))

Expand Down
4 changes: 3 additions & 1 deletion otter/test_files/ok_test.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -90,7 +90,9 @@ def run(self, global_environment):
for i, test_case in enumerate(self.test_cases):
passed, result = run_doctest(self.name + ' ' + str(i), test_case.body, global_environment)
if passed:
result = 'Test case passed!'
result = '✅ Test case passed'
else:
result = '❌ Test case failed\n' + result

self.test_case_results.append(TestCaseResult(
test_case = test_case,
Expand Down
6 changes: 4 additions & 2 deletions test/test_check.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,7 @@ def test_otter_check_script():
q1 results: All test cases passed!
q2 results:
q2 - 1 result:
❌ Test case failed
Trying:
1 == 1
Expecting:
Expand All @@ -74,7 +75,7 @@ def test_otter_check_script():
True
q2 - 2 result:
Test case passed!
✅ Test case passed
q3 results: All test cases passed!
q4 results: All test cases passed!
q5 results: All test cases passed!"""), \
Expand Down Expand Up @@ -118,6 +119,7 @@ def test_otter_check_notebook():
q1 results: All test cases passed!
q2 results:
q2 - 1 result:
❌ Test case failed
Trying:
1 == 1
Expecting:
Expand All @@ -132,7 +134,7 @@ def test_otter_check_notebook():
True
q2 - 2 result:
Test case passed!
✅ Test case passed
q3 results: All test cases passed!
q4 results: All test cases passed!
q5 results: All test cases passed!"""), \
Expand Down
3 changes: 2 additions & 1 deletion test/test_notebook.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,7 @@ def negate(x):
"q2": dedent("""\
q2 results:
q2 - 1 result:
❌ Test case failed
Trying:
1 == 1
Expecting:
Expand All @@ -70,7 +71,7 @@ def negate(x):
True
q2 - 2 result:
Test case passed!
✅ Test case passed
""").strip(),
"q3": 'q3 results: All test cases passed!',
"q4": 'q4 results: All test cases passed!',
Expand Down
6 changes: 3 additions & 3 deletions test/test_run.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -56,7 +56,7 @@ def expected_results():
{
"name": "Public Tests",
"visibility": "visible",
"output": "q1 results: All test cases passed!\n\nq2 results:\n q2 - 1 result:\n Trying:\n negate(True)\n Expecting:\n False\n **********************************************************************\n Line 2, in q2 0\n Failed example:\n negate(True)\n Expected:\n False\n Got:\n True\n\n q2 - 2 result:\n Trying:\n negate(False)\n Expecting:\n True\n **********************************************************************\n Line 2, in q2 1\n Failed example:\n negate(False)\n Expected:\n True\n Got:\n False\n\nq3 results: All test cases passed!\n\nq4 results: All test cases passed!\n\nq6 results: All test cases passed!\n\nq7 results: All test cases passed!",
"output": "q1 results: All test cases passed!\n\nq2 results:\n q2 - 1 result:\n ❌ Test case failed\n Trying:\n negate(True)\n Expecting:\n False\n **********************************************************************\n Line 2, in q2 0\n Failed example:\n negate(True)\n Expected:\n False\n Got:\n True\n\n q2 - 2 result:\n ❌ Test case failed\n Trying:\n negate(False)\n Expecting:\n True\n **********************************************************************\n Line 2, in q2 1\n Failed example:\n negate(False)\n Expected:\n True\n Got:\n False\n\nq3 results: All test cases passed!\n\nq4 results: All test cases passed!\n\nq6 results: All test cases passed!\n\nq7 results: All test cases passed!",
"status": "passed",
},
{
Expand All @@ -71,7 +71,7 @@ def expected_results():
"score": 0,
"max_score": 2.0,
"visibility": "hidden",
"output": "q2 results:\n q2 - 1 result:\n Trying:\n negate(True)\n Expecting:\n False\n **********************************************************************\n Line 2, in q2 0\n Failed example:\n negate(True)\n Expected:\n False\n Got:\n True\n\n q2 - 2 result:\n Trying:\n negate(False)\n Expecting:\n True\n **********************************************************************\n Line 2, in q2 1\n Failed example:\n negate(False)\n Expected:\n True\n Got:\n False\n\n q2 - 3 result:\n Trying:\n negate(\"\")\n Expecting:\n True\n **********************************************************************\n Line 2, in q2 2\n Failed example:\n negate(\"\")\n Expected:\n True\n Got:\n ''\n\n q2 - 4 result:\n Trying:\n negate(1)\n Expecting:\n False\n **********************************************************************\n Line 2, in q2 3\n Failed example:\n negate(1)\n Expected:\n False\n Got:\n 1"
"output": "q2 results:\n q2 - 1 result:\n ❌ Test case failed\n Trying:\n negate(True)\n Expecting:\n False\n **********************************************************************\n Line 2, in q2 0\n Failed example:\n negate(True)\n Expected:\n False\n Got:\n True\n\n q2 - 2 result:\n ❌ Test case failed\n Trying:\n negate(False)\n Expecting:\n True\n **********************************************************************\n Line 2, in q2 1\n Failed example:\n negate(False)\n Expected:\n True\n Got:\n False\n\n q2 - 3 result:\n ❌ Test case failed\n Trying:\n negate(\"\")\n Expecting:\n True\n **********************************************************************\n Line 2, in q2 2\n Failed example:\n negate(\"\")\n Expected:\n True\n Got:\n ''\n\n q2 - 4 result:\n ❌ Test case failed\n Trying:\n negate(1)\n Expecting:\n False\n **********************************************************************\n Line 2, in q2 3\n Failed example:\n negate(1)\n Expected:\n False\n Got:\n 1"
},
{
"name": "q3",
Expand All @@ -92,7 +92,7 @@ def expected_results():
"score": 2.5,
"max_score": 5.0,
"visibility": "hidden",
"output": "q6 results:\n q6 - 1 result:\n Test case passed!\n\n q6 - 2 result:\n Trying:\n fib = fiberator()\n Expecting nothing\n ok\n Trying:\n for _ in range(10):\n print(next(fib))\n Expecting:\n 0\n 1\n 1\n 2\n 3\n 5\n 8\n 13\n 21\n 34\n **********************************************************************\n Line 3, in q6 1\n Failed example:\n for _ in range(10):\n print(next(fib))\n Expected:\n 0\n 1\n 1\n 2\n 3\n 5\n 8\n 13\n 21\n 34\n Got:\n 0\n 1\n 1\n 1\n 2\n 3\n 5\n 8\n 13\n 21"
"output": "q6 results:\n q6 - 1 result:\n ✅ Test case passed\n\n q6 - 2 result:\n ❌ Test case failed\n Trying:\n fib = fiberator()\n Expecting nothing\n ok\n Trying:\n for _ in range(10):\n print(next(fib))\n Expecting:\n 0\n 1\n 1\n 2\n 3\n 5\n 8\n 13\n 21\n 34\n **********************************************************************\n Line 3, in q6 1\n Failed example:\n for _ in range(10):\n print(next(fib))\n Expected:\n 0\n 1\n 1\n 2\n 3\n 5\n 8\n 13\n 21\n 34\n Got:\n 0\n 1\n 1\n 1\n 2\n 3\n 5\n 8\n 13\n 21"
},
{
"name": "q7",
Expand Down

0 comments on commit e281f10

Please sign in to comment.