Skip to content

Commit 6144181

Browse files
LoumstarLoumstar
committed
Completed commands unit tests
1 parent 358deb1 commit 6144181

File tree

6 files changed

+399
-36
lines changed

6 files changed

+399
-36
lines changed

‎requirements.txt‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,6 @@
11
jsonschema
22
requests
33
evaluation-function-utils
4-
typing_extensions
4+
typing_extensions
5+
black
6+
autopep8

‎tests/commands.py‎

Lines changed: 315 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -1,56 +1,348 @@
11
import unittest
2+
from typing import Any
23

3-
from ..tools import commands # noqa
4+
from ..tools import commands, parse, validate
5+
from ..tools.utils import JsonType
6+
7+
8+
def evaluation_function(
9+
response: Any, answer: Any, params: JsonType
10+
) -> JsonType:
11+
if params.get("raise", False):
12+
raise ValueError("raised")
13+
14+
force_true = params.get("force", False)
15+
16+
return {"is_correct": (response == answer) or force_true}
417

518

619
class TestCommandsModule(unittest.TestCase):
20+
def setUp(self) -> None:
21+
commands.evaluation_function = evaluation_function
22+
return super().setUp()
23+
24+
def tearDown(self) -> None:
25+
commands.evaluation_function = None
26+
return super().tearDown()
27+
728
def test_valid_eval_command(self):
8-
...
29+
event = {
30+
"body": {"response": "hello", "answer": "world", "params": {}}
31+
}
32+
response = commands.evaluate(event)
33+
34+
self.assertIn("result", response)
35+
self.assertIn("is_correct", response["result"]) # type: ignore
936

1037
def test_invalid_eval_args_raises_parse_error(self):
11-
...
38+
event = {"headers": "any", "other": "params"}
39+
40+
with self.assertRaises(parse.ParseError) as e:
41+
commands.evaluate(event)
42+
43+
self.assertEqual(
44+
e.exception.message, "No data supplied in request body."
45+
)
1246

1347
def test_invalid_eval_schema_raises_validation_error(self):
14-
...
48+
event = {"body": {"response": "hello", "params": {}}}
49+
50+
with self.assertRaises(validate.ValidationError) as e:
51+
commands.evaluate(event)
52+
53+
self.assertEqual(
54+
e.exception.message,
55+
"Failed to validate body against the evaluation schema.",
56+
)
1557

1658
def test_single_feedback_case(self):
17-
...
59+
event = {
60+
"body": {
61+
"response": "hello",
62+
"answer": "hello",
63+
"params": {
64+
"cases": [
65+
{
66+
"answer": "other",
67+
"feedback": "should be 'other'.",
68+
"mark": 0,
69+
}
70+
]
71+
},
72+
}
73+
}
74+
75+
response = commands.evaluate(event)
76+
result = response["result"] # type: ignore
77+
78+
self.assertTrue(result["is_correct"])
79+
self.assertNotIn("match", result)
80+
self.assertNotIn("feedback", result)
1881

1982
def test_single_feedback_case_match(self):
20-
...
83+
event = {
84+
"body": {
85+
"response": "hello",
86+
"answer": "world",
87+
"params": {
88+
"cases": [
89+
{
90+
"answer": "hello",
91+
"feedback": "should be 'hello'.",
92+
}
93+
]
94+
},
95+
}
96+
}
97+
98+
response = commands.evaluate(event)
99+
result = response["result"] # type: ignore
100+
101+
self.assertFalse(result["is_correct"])
102+
self.assertEqual(result["match"], 0)
103+
self.assertEqual(result["feedback"], "should be 'hello'.")
21104

22105
def test_case_warning_data_structure(self):
23-
...
106+
event = {
107+
"body": {
108+
"response": "hello",
109+
"answer": "world",
110+
"params": {
111+
"cases": [
112+
{
113+
"feedback": "should be 'hello'.",
114+
}
115+
]
116+
},
117+
}
118+
}
119+
120+
response = commands.evaluate(event)
121+
result = response["result"] # type: ignore
122+
123+
self.assertIn("warnings", result)
124+
warning = result["warnings"].pop() # type: ignore
125+
126+
self.assertDictEqual(
127+
warning,
128+
{"case": 0, "message": "Missing answer/feedback field"},
129+
)
24130

25131
def test_multiple_feedback_cases_single_match(self):
26-
...
132+
event = {
133+
"body": {
134+
"response": "yes",
135+
"answer": "world",
136+
"params": {
137+
"cases": [
138+
{
139+
"answer": "hello",
140+
"feedback": "should be 'hello'.",
141+
},
142+
{
143+
"answer": "yes",
144+
"feedback": "should be 'yes'.",
145+
},
146+
{
147+
"answer": "no",
148+
"feedback": "should be 'no'.",
149+
},
150+
]
151+
},
152+
}
153+
}
154+
155+
response = commands.evaluate(event)
156+
result = response["result"] # type: ignore
157+
158+
self.assertFalse(result["is_correct"])
159+
self.assertEqual(result["match"], 1)
160+
self.assertEqual(result["feedback"], "should be 'yes'.")
27161

28162
def test_multiple_feedback_cases_multiple_matches(self):
29-
...
163+
event = {
164+
"body": {
165+
"response": "yes",
166+
"answer": "world",
167+
"params": {
168+
"cases": [
169+
{
170+
"answer": "hello",
171+
"feedback": "should be 'hello'.",
172+
"params": {"force": True},
173+
},
174+
{
175+
"answer": "yes",
176+
"feedback": "should be 'yes'.",
177+
},
178+
{
179+
"answer": "no",
180+
"feedback": "should be 'no'.",
181+
},
182+
]
183+
},
184+
}
185+
}
186+
187+
response = commands.evaluate(event)
188+
result = response["result"] # type: ignore
189+
190+
self.assertFalse(result["is_correct"])
191+
self.assertEqual(result["match"], 0)
192+
self.assertEqual(result["feedback"], "should be 'hello'.")
193+
194+
def test_case_params_overwrite_eval_params(self):
195+
event = {
196+
"body": {
197+
"response": "hello",
198+
"answer": "world",
199+
"params": {
200+
"force": True,
201+
"cases": [
202+
{
203+
"answer": "yes",
204+
"feedback": "should be 'yes'.",
205+
"params": {"force": False},
206+
}
207+
],
208+
},
209+
}
210+
}
211+
212+
response = commands.evaluate(event)
213+
result = response["result"] # type: ignore
214+
215+
self.assertTrue(result["is_correct"])
216+
self.assertNotIn("match", result)
217+
self.assertNotIn("feedback", result)
30218

31219
def test_invalid_case_entry_doesnt_raise_exception(self):
32-
...
220+
event = {
221+
"body": {
222+
"response": "hello",
223+
"answer": "world",
224+
"params": {
225+
"cases": [
226+
{
227+
"answer": "hello",
228+
"feedback": "should be 'hello'.",
229+
"params": {"raise": True},
230+
}
231+
]
232+
},
233+
}
234+
}
235+
236+
response = commands.evaluate(event)
237+
result = response["result"] # type: ignore
238+
239+
self.assertIn("warnings", result)
240+
warning = result["warnings"].pop() # type: ignore
241+
242+
self.assertDictEqual(
243+
warning,
244+
{
245+
"case": 0,
246+
"message": "An exception was raised while executing "
247+
"the evaluation function.",
248+
"detail": "raised",
249+
},
250+
)
33251

34252
def test_multiple_matched_cases_are_combined_and_warned(self):
35-
...
253+
event = {
254+
"body": {
255+
"response": "yes",
256+
"answer": "world",
257+
"params": {
258+
"cases": [
259+
{
260+
"answer": "hello",
261+
"feedback": "should be 'hello'.",
262+
"params": {"force": True},
263+
},
264+
{
265+
"answer": "yes",
266+
"feedback": "should be 'yes'.",
267+
},
268+
{
269+
"answer": "no",
270+
"feedback": "should be 'no'.",
271+
},
272+
]
273+
},
274+
}
275+
}
36276

37-
def test_overriding_eval_feedback_in_case(self):
38-
...
277+
response = commands.evaluate(event)
278+
result = response["result"] # type: ignore
39279

40-
def test_overriding_is_correct_with_case_mark(self):
41-
...
280+
self.assertIn("warnings", result)
281+
warning = result["warnings"].pop() # type: ignore
42282

43-
def test_valid_preview_command(self):
44-
...
283+
self.assertDictEqual(
284+
warning,
285+
{
286+
"message": "Cases 0, 1 were matched. "
287+
"Only the first one's feedback was returned",
288+
},
289+
)
290+
291+
def test_overriding_eval_feedback_to_correct_case(self):
292+
event = {
293+
"body": {
294+
"response": "hello",
295+
"answer": "world",
296+
"params": {
297+
"cases": [
298+
{
299+
"answer": "hello",
300+
"feedback": "should be 'hello'.",
301+
"mark": 1,
302+
}
303+
]
304+
},
305+
}
306+
}
307+
308+
response = commands.evaluate(event)
309+
result = response["result"] # type: ignore
310+
311+
self.assertTrue(result["is_correct"])
312+
self.assertEqual(result["match"], 0)
313+
self.assertEqual(result["feedback"], "should be 'hello'.")
45314

46-
def test_invalid_preview_args_raises_parse_error(self):
47-
...
315+
def test_overriding_eval_feedback_to_incorrect_case(self):
316+
event = {
317+
"body": {
318+
"response": "hello",
319+
"answer": "hello",
320+
"params": {
321+
"cases": [
322+
{
323+
"answer": "hello",
324+
"feedback": "should be 'hello'.",
325+
"mark": 0,
326+
}
327+
]
328+
},
329+
}
330+
}
331+
332+
response = commands.evaluate(event)
333+
result = response["result"] # type: ignore
334+
335+
self.assertFalse(result["is_correct"])
336+
self.assertEqual(result["match"], 0)
337+
self.assertEqual(result["feedback"], "should be 'hello'.")
338+
339+
def test_valid_preview_command(self):
340+
event = {"body": {"response": "hello"}}
48341

49-
def test_invalid_preview_schema_raises_validation_error(self):
50-
...
342+
response = commands.preview(event)
343+
result = response["result"] # type: ignore
51344

52-
def test_handle_healthcheck_command(self):
53-
...
345+
self.assertEqual(result["preview"], "hello")
54346

55347

56348
if __name__ == "__main__":

0 commit comments

Comments
 (0)