add rich feedback handling

Author: RabidSheep55

handler.py

@@ -51,16 +51,12 @@ def handle_eval_command(event):
     if request_error:
         return {"error": request_error}
 
-    try:
-        response = body["response"]
-        answer = body["answer"]
-
-        params = body.get("params", dict())
+    response = body["response"]
+    answer = body["answer"]
+    params = body.get("params", dict())
 
-        return {
-            "command": "eval",
-            "result": evaluation_function(response, answer, params)
-        }
+    try:
+        result = evaluation_function(response, answer, params)
 
     # Catch the custom EvaluationException (from evaluation_function_utils) first
     except EvaluationException as e:
@@ -75,6 +71,96 @@ def handle_eval_command(event):
             }
         }
 
+    # Result already contains feedback generated inside the custom function
+    if "feedback" in result:
+        return {"command": "eval", "result": result}
+
+    # If a list of "cases" wasn't provided, we don't have any other way to get feedback
+    cases = params.get("cases", [])
+    if len(cases) == 0:
+        return {"command": "eval", "result": result}
+
+    # Determine what feedback to provide based on cases
+    feedback, warnings = feedback_from_cases(response, params, cases)
+    if feedback:
+        result["feedback"] = feedback
+
+    # Add warnings out output if any were encountered
+    if len(warnings) != 0:
+        result["warnings"] = warnings
+
+    return {"command": "eval", "result": result}
+
+
+def feedback_from_cases(response, params, cases):
+    """ 
+    Attempt to find the correct feedback from a list of cases.
+    Returns a "feedback" string, and optional list of warnings
+    """
+
+    # A list of "cases" was provided, try matching to each of them
+    matches = []
+    warnings = []
+    for i, case in enumerate(cases):
+        # Validate the case block has an answer and feedback
+        if 'answer' not in case:
+            warnings += [{"case": i, "message": "Missing answer field"}]
+            continue
+
+        if 'feedback' not in case:
+            warnings += [{"case": i, "message": "Missing feedback field"}]
+            continue
+
+        # Merge current evaluation params with any specified in case
+        case_params = case.get('params', {})
+
+        # Run the evaluation function based on this case's answer
+        try:
+            res = evaluation_function(response, case.get('answer'), {
+                **params,
+                **case_params
+            })
+
+        except EvaluationException as e:
+            warnings += [{"case": i, **e.error_dict}]
+            continue
+
+        except Exception as e:
+            warnings += [{
+                "case": i,
+                "message":
+                "An exception was raised while executing the evaluation function.",
+                "detail": str(e) if str(e) != "" else repr(e)
+            }]
+            continue
+
+        # Function should always return an 'is_correct' if no errors were raised
+        if not 'is_correct' in res:
+            warnings += [{
+                "case": i,
+                "message": "is_correct missing from function output"
+            }]
+            continue
+
+        # This case matches the response, add it's index to the list of matches
+        if res.get('is_correct') == True:
+            matches += [i]
+
+    if len(matches) == 0:
+        return '', warnings
+
+    if len(matches) == 1:
+        # warnings += [{"case": matches[0]}]
+        return cases[matches[0]]['feedback'], warnings
+
+    else:
+        s = ', '.join([str(m) for m in matches])
+        warnings += [{
+            "message":
+            f"Cases {s} were matched. Only the first one's feedback was returned"
+        }]
+        return cases[matches[0]]['feedback'], warnings
+
 
 """
     Main Handler Function