Added separate set of tests for healthcheck

Author: Loumstar

Dockerfile

@@ -25,4 +25,4 @@ COPY tests/*.py ./app/tests/
 COPY tools/*.py ./app/tools/
 
 # Request-response-schemas repo/branch to use for validation
-ENV SCHEMAS_URL = https://raw.githubusercontent.com/lambda-feedback/request-response-schemas/579-adding-preview-command
+ENV SCHEMAS_URL=https://raw.githubusercontent.com/lambda-feedback/request-response-schemas/579-adding-preview-command

requirements.txt

@@ -3,4 +3,6 @@ requests
 evaluation-function-utils
 typing_extensions
 black
-autopep8
+autopep8
+pytest
+python-dotenv

tests/__init__.py

@@ -1,5 +0,0 @@
-from .commands import TestCommandsModule  # noqa
-from .docs import TestDocsModule  # noqa
-from .parse import TestParseModule  # noqa
-from .requests import TestRequestValidation  # noqa
-from .responses import TestResponseValidation  # noqa

tests/commands.py

@@ -17,6 +17,9 @@ def evaluation_function(
 
 
 class TestCommandsModule(unittest.TestCase):
+    def __init__(self, methodName: str = "runTest") -> None:
+        super().__init__(methodName)
+
     def setUp(self) -> None:
         commands.evaluation_function = evaluation_function
         return super().setUp()
@@ -344,6 +347,35 @@ def test_valid_preview_command(self):
 
         self.assertEqual(result["preview"], "hello")
 
+    def test_invalid_preview_args_raises_parse_error(self):
+        event = {"headers": "any", "other": "params"}
+
+        with self.assertRaises(parse.ParseError) as e:
+            commands.preview(event)
+
+        self.assertEqual(
+            e.exception.message, "No data supplied in request body."
+        )
+
+    def test_invalid_preview_schema_raises_validation_error(self):
+        event = {"body": {"response": "hello", "answer": "hello"}}
+
+        with self.assertRaises(validate.ValidationError) as e:
+            commands.preview(event)
+
+        self.assertEqual(
+            e.exception.message,
+            "Failed to validate body against the preview schema.",
+        )
+
+    def test_healthcheck(self):
+        response = commands.healthcheck()
+
+        self.assertIn("result", response)
+        result = response["result"]  # type: ignore
+
+        self.assertIn("tests_passed", result)
+
 
 if __name__ == "__main__":
     unittest.main()

tests/docs.py

@@ -32,7 +32,12 @@ def test_handling_missing_doc(self):
             "tests/non-existent-doc.md missing from evaluation function files",
         )
 
-    """Can't test available dev docs without the layer above."""
+    """
+    Can't test available dev docs without the layer above.
+
+    Instead, we can only test that missing docs are correctly handled.
+    However, we test for available docs in the healthcheck smoke tests.
+    """
 
     def test_handling_missing_user_docs(self):
         result = docs.user()

tests/handling.py

@@ -114,10 +114,6 @@ def test_healthcheck(self):
         self.assertEqual(response.get("command"), "healthcheck")
         self.assertIn("result", response)
 
-        result = response.get("result")
-
-        self.assertTrue(result["tests_passed"])  # type: ignore
-
     def test_invalid_command(self):
         event = {
             "random": "metadata",

tools/commands.py

@@ -14,7 +14,7 @@
 from .validate import ReqBodyValidators
 
 try:
-    from ..evaluate import evaluation_function  # type: ignore
+    from ..evaluation import evaluation_function  # type: ignore
 
 except ImportError:
     evaluation_function: Optional[EvaluationFunctionType] = None

tools/healthcheck.py

@@ -7,11 +7,7 @@
 
 from typing_extensions import NotRequired
 
-from ..tests.commands import TestCommandsModule
-from ..tests.docs import TestDocsModule
-from ..tests.parse import TestParseModule
-from ..tests.requests import TestRequestValidation
-from ..tests.responses import TestResponseValidation
+from .smoke_tests import SmokeTests
 
 try:
     from ..evaluation_tests import TestEvaluationFunction  # type: ignore
@@ -130,17 +126,13 @@ def healthcheck() -> HealthcheckJsonTestResult:
     """
     # Redirect stderr stream to null to prevent logging unittest results
     no_stream = open(os.devnull, "w")
-    sys.stderr = no_stream
+    # sys.stderr = no_stream
 
     # Create a test loader and test runner instance
     loader = unittest.TestLoader()
 
     cases = (
-        TestRequestValidation,
-        TestResponseValidation,
-        TestCommandsModule,
-        TestParseModule,
-        TestDocsModule,
+        SmokeTests,
         TestEvaluationFunction,
         TestPreviewFunction,
     )

tools/smoke_tests.py

@@ -0,0 +1,99 @@
+import inspect
+import unittest
+
+import jsonschema
+
+from . import docs, validate
+
+
+class SmokeTests(unittest.TestCase):
+    """Tests to check that all functions, files and tests are defined
+    in the evaluation function by the user.
+
+    Base Evaluation Function tests are no longer applied in the healthcheck,
+    instead we only check that the necessary functions, classes and files
+    are defined (here) and the user-defined tests pass.
+
+    Base Evaluation Function tests are only applied when pushing to Docker.
+
+    This test case also checks that all schemas are working appropriately.
+    """
+
+    def test_import_evaluation_function(self):
+        from .. import evaluation  # type: ignore
+
+        self.assertTrue(inspect.ismodule(evaluation))
+        self.assertIsNotNone(evaluation.evaluation_function)
+
+    @unittest.skip("Ignored until all functions are updated with preview.")
+    def test_import_preview_function(self):
+        from .. import preview  # type: ignore
+
+        self.assertTrue(inspect.ismodule(preview))
+        self.assertIsNotNone(preview.preview_function)
+
+    def test_import_evaluation_tests(self):
+        from .. import evaluation_tests  # type: ignore
+
+        self.assertTrue(inspect.ismodule(evaluation_tests))
+        self.assertIsNotNone(evaluation_tests.TestEvaluationFunction)
+
+    @unittest.skip("Ignored until all functions are updated with preview.")
+    def test_import_preview_tests(self):
+        from .. import preview_tests  # type: ignore
+
+        self.assertTrue(inspect.ismodule(preview_tests))
+        self.assertIsNotNone(preview_tests.TestPreviewFunction)
+
+    def test_load_dev_docs(self):
+        result = docs.dev()
+
+        self.assertEqual(result["statusCode"], 200)
+        self.assertDictEqual(
+            result["headers"], {"Content-Type": "application/octet-stream"}
+        )
+        self.assertTrue(result["isBase64Encoded"])
+
+    def test_load_user_docs(self):
+        result = docs.user()
+
+        self.assertEqual(result["statusCode"], 200)
+        self.assertDictEqual(
+            result["headers"], {"Content-Type": "application/octet-stream"}
+        )
+        self.assertTrue(result["isBase64Encoded"])
+
+    def test_load_eval_req_schema(self):
+        schema = validate.load_validator_from_url(
+            validate.ReqBodyValidators.EVALUATION
+        )
+
+        self.assertIsInstance(schema, jsonschema.Draft7Validator)
+
+    def test_load_preview_req_schema(self):
+        schema = validate.load_validator_from_url(
+            validate.ReqBodyValidators.PREVIEW
+        )
+
+        self.assertIsInstance(schema, jsonschema.Draft7Validator)
+
+    def test_load_eval_res_schema(self):
+        schema = validate.load_validator_from_url(
+            validate.ResBodyValidators.EVALUATION
+        )
+
+        self.assertIsInstance(schema, jsonschema.Draft7Validator)
+
+    def test_load_preview_res_schema(self):
+        schema = validate.load_validator_from_url(
+            validate.ResBodyValidators.PREVIEW
+        )
+
+        self.assertIsInstance(schema, jsonschema.Draft7Validator)
+
+    def test_load_health_res_schema(self):
+        schema = validate.load_validator_from_url(
+            validate.ResBodyValidators.HEALTHCHECK
+        )
+
+        self.assertIsInstance(schema, jsonschema.Draft7Validator)

tools/validate.py

@@ -3,10 +3,13 @@
 import os
 from typing import Dict, List, TypedDict, Union
 
+import dotenv
 import jsonschema
 import jsonschema.exceptions
 import requests
 
+dotenv.load_dotenv()
+
 
 class SchemaErrorThrown(TypedDict):
     """Detail object returned in the error response for schema exceptions."""
@@ -52,7 +55,9 @@ class ResBodyValidators(enum.Enum):
 
 
 @functools.lru_cache
-def load_validator_from_url(validator_enum: BodyValidators):
+def load_validator_from_url(
+    validator_enum: BodyValidators,
+) -> jsonschema.Draft7Validator:
     """Loads a json schema for body validations.
 
     Args:
@@ -67,12 +72,19 @@ def load_validator_from_url(validator_enum: BodyValidators):
     schemas_url = os.environ.get("SCHEMAS_URL")
 
     if schemas_url is None:
-        raise ValueError("Schema URL is not defined in base layer.")
+        raise RuntimeError("Schema URL is not defined in base layer.")
 
     schema_url = os.path.join(schemas_url, validator_enum.value)
 
-    schema = requests.get(schema_url).json()
-    return jsonschema.Draft7Validator(schema)
+    response = requests.get(schema_url)
+
+    if response.status_code != 200:
+        raise RuntimeError(
+            f"Failed to get {validator_enum.name.lower()} "
+            f"schema (status code {response.status_code})."
+        )
+
+    return jsonschema.Draft7Validator(response.json())
 
 
 def body(body: Union[Dict, TypedDict], validator_enum: BodyValidators) -> None: