From ff1e4874aca5966333697fd485ae84a7cf2bfa81 Mon Sep 17 00:00:00 2001 From: BethanyG Date: Tue, 11 Feb 2025 16:55:42 -0800 Subject: [PATCH] Changed sort order for concept exercises with failed subtests and rerecoreded golden output for them, --- runner/data.py | 26 +++- .../results.json | 94 ++++++------ .../results.json | 16 +- .../example-has-stdout-and-tasks/results.json | 10 +- .../results.json | 42 +++--- ...tial_failure_with_subtests_and_stdout_2.py | 86 +++++++++++ ...failure_with_subtests_and_stdout_2_test.py | 141 ++++++++++++++++++ .../results.json | 66 ++++++++ .../results.json | 94 ++++++------ .../results.json | 32 ++-- .../results.json | 8 +- 11 files changed, 462 insertions(+), 153 deletions(-) create mode 100644 test/example-partial-failure-with-subtests-and-stdout-2/example_partial_failure_with_subtests_and_stdout_2.py create mode 100644 test/example-partial-failure-with-subtests-and-stdout-2/example_partial_failure_with_subtests_and_stdout_2_test.py create mode 100644 test/example-partial-failure-with-subtests-and-stdout-2/results.json diff --git a/runner/data.py b/runner/data.py index 24a1cc9..133a6a5 100644 --- a/runner/data.py +++ b/runner/data.py @@ -153,15 +153,31 @@ def _factory(items): def as_json(self): """ - Trim off the TestClass name and test_ prefix from each test_name. - Replace underscores with spaces for more human-readable strings. - Sort the current tests array by task_id and then Dump all results to formatted JSON. + - Trim off the TestClass name and test_ prefix from each test_name. + - Replace underscores with spaces for more human-readable strings. + - Add a sort order signifier (~) to parent tests with subtests. + (~) ensures parent tests sort to last position. + + - If it is a concept exercise, sort the current tests array by + task_id and variation# then Dump all results to formatted JSON. """ + + trim_name = compile(r'^(.+)(Test\.test_)') results = asdict(self, dict_factory=self._factory) + concept_exercise = False for item in results["tests"]: - item["name"] = sub(trim_name, '\\1 > ', item["name"]).replace('_', ' ') - results["tests"] = sorted(results["tests"], key= lambda item: item["task_id"]) + if "[variation" not in item["name"] and item["task_id"] > 0: + item["name"] = sub(trim_name, '\\1 > ', item["name"]).replace('_', ' ') + "~" + concept_exercise = True + else: + item["name"] = sub(trim_name, '\\1 > ', item["name"]).replace('_', ' ') + + if concept_exercise: + results["tests"] = sorted(results["tests"], key= lambda item: (item["task_id"], item["name"])) + else: + results["tests"] = sorted(results["tests"], key=lambda item: (item["task_id"])) + return dumps(results, indent=2) diff --git a/test/example-all-fail-tasks-and-subtests/results.json b/test/example-all-fail-tasks-and-subtests/results.json index 795ba7e..ea67d2c 100644 --- a/test/example-all-fail-tasks-and-subtests/results.json +++ b/test/example-all-fail-tasks-and-subtests/results.json @@ -2,48 +2,6 @@ "version": 3, "status": "fail", "tests": [ - { - "name": "ExampleAllFail > abc", - "status": "fail", - "message": "One or more variations of this test failed. Details can be found under each [variant#].", - "test_code": "input_data = ['frog', 'fish', 'coconut', 'pineapple', 'carrot', 'cucumber', 'grass', 'tree']\nresult_data = [(\"Hello, World!\", param) for param in input_data]\n\nfor variant, (param, result) in enumerate(zip(input_data, result_data), start=1):\n failure_msg=f'Expected: {result} but got something else instead.'\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result, msg=failure_msg)", - "task_id": 1 - }, - { - "name": "ExampleAllFail > hello", - "status": "fail", - "message": "One or more variations of this test failed. Details can be found under each [variant#].", - "test_code": "input_data = [15, 23, 33, 39]\nresult_data = [(\"Hello, World!\", param) for param in input_data]\n\nfor variant, (param, result) in enumerate(zip(input_data, result_data), start=1):\n failure_msg=f'Expected: {result} but got something else instead.'\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result, msg=failure_msg)", - "task_id": 1 - }, - { - "name": "ExampleAllFail > hello [variation #1]", - "status": "fail", - "message": "AssertionError: 'Goodbye!' != ('Hello, World!', 15) : Expected: ('Hello, World!', 15) but got something else instead.", - "test_code": "input_data = [15, 23, 33, 39]\nresult_data = [(\"Hello, World!\", param) for param in input_data]\n\nfor variant, (param, result) in enumerate(zip(input_data, result_data), start=1):\n failure_msg=f'Expected: {result} but got something else instead.'\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result, msg=failure_msg)", - "task_id": 1 - }, - { - "name": "ExampleAllFail > hello [variation #2]", - "status": "fail", - "message": "AssertionError: 'Goodbye!' != ('Hello, World!', 23) : Expected: ('Hello, World!', 23) but got something else instead.", - "test_code": "input_data = [15, 23, 33, 39]\nresult_data = [(\"Hello, World!\", param) for param in input_data]\n\nfor variant, (param, result) in enumerate(zip(input_data, result_data), start=1):\n failure_msg=f'Expected: {result} but got something else instead.'\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result, msg=failure_msg)", - "task_id": 1 - }, - { - "name": "ExampleAllFail > hello [variation #3]", - "status": "fail", - "message": "AssertionError: 'Goodbye!' != ('Hello, World!', 33) : Expected: ('Hello, World!', 33) but got something else instead.", - "test_code": "input_data = [15, 23, 33, 39]\nresult_data = [(\"Hello, World!\", param) for param in input_data]\n\nfor variant, (param, result) in enumerate(zip(input_data, result_data), start=1):\n failure_msg=f'Expected: {result} but got something else instead.'\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result, msg=failure_msg)", - "task_id": 1 - }, - { - "name": "ExampleAllFail > hello [variation #4]", - "status": "fail", - "message": "AssertionError: 'Goodbye!' != ('Hello, World!', 39) : Expected: ('Hello, World!', 39) but got something else instead.", - "test_code": "input_data = [15, 23, 33, 39]\nresult_data = [(\"Hello, World!\", param) for param in input_data]\n\nfor variant, (param, result) in enumerate(zip(input_data, result_data), start=1):\n failure_msg=f'Expected: {result} but got something else instead.'\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result, msg=failure_msg)", - "task_id": 1 - }, { "name": "ExampleAllFail > abc [variation #1]", "status": "fail", @@ -101,18 +59,46 @@ "task_id": 1 }, { - "name": "ExampleAllFailOther > dummy", + "name": "ExampleAllFail > abc~", "status": "fail", "message": "One or more variations of this test failed. Details can be found under each [variant#].", "test_code": "input_data = ['frog', 'fish', 'coconut', 'pineapple', 'carrot', 'cucumber', 'grass', 'tree']\nresult_data = [(\"Hello, World!\", param) for param in input_data]\n\nfor variant, (param, result) in enumerate(zip(input_data, result_data), start=1):\n failure_msg=f'Expected: {result} but got something else instead.'\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result, msg=failure_msg)", - "task_id": 2 + "task_id": 1 + }, + { + "name": "ExampleAllFail > hello [variation #1]", + "status": "fail", + "message": "AssertionError: 'Goodbye!' != ('Hello, World!', 15) : Expected: ('Hello, World!', 15) but got something else instead.", + "test_code": "input_data = [15, 23, 33, 39]\nresult_data = [(\"Hello, World!\", param) for param in input_data]\n\nfor variant, (param, result) in enumerate(zip(input_data, result_data), start=1):\n failure_msg=f'Expected: {result} but got something else instead.'\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result, msg=failure_msg)", + "task_id": 1 + }, + { + "name": "ExampleAllFail > hello [variation #2]", + "status": "fail", + "message": "AssertionError: 'Goodbye!' != ('Hello, World!', 23) : Expected: ('Hello, World!', 23) but got something else instead.", + "test_code": "input_data = [15, 23, 33, 39]\nresult_data = [(\"Hello, World!\", param) for param in input_data]\n\nfor variant, (param, result) in enumerate(zip(input_data, result_data), start=1):\n failure_msg=f'Expected: {result} but got something else instead.'\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result, msg=failure_msg)", + "task_id": 1 }, { - "name": "ExampleAllFailOther > hello", + "name": "ExampleAllFail > hello [variation #3]", + "status": "fail", + "message": "AssertionError: 'Goodbye!' != ('Hello, World!', 33) : Expected: ('Hello, World!', 33) but got something else instead.", + "test_code": "input_data = [15, 23, 33, 39]\nresult_data = [(\"Hello, World!\", param) for param in input_data]\n\nfor variant, (param, result) in enumerate(zip(input_data, result_data), start=1):\n failure_msg=f'Expected: {result} but got something else instead.'\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result, msg=failure_msg)", + "task_id": 1 + }, + { + "name": "ExampleAllFail > hello [variation #4]", + "status": "fail", + "message": "AssertionError: 'Goodbye!' != ('Hello, World!', 39) : Expected: ('Hello, World!', 39) but got something else instead.", + "test_code": "input_data = [15, 23, 33, 39]\nresult_data = [(\"Hello, World!\", param) for param in input_data]\n\nfor variant, (param, result) in enumerate(zip(input_data, result_data), start=1):\n failure_msg=f'Expected: {result} but got something else instead.'\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result, msg=failure_msg)", + "task_id": 1 + }, + { + "name": "ExampleAllFail > hello~", "status": "fail", "message": "One or more variations of this test failed. Details can be found under each [variant#].", - "test_code": "input_data = [1, 2, 5, 10]\nresult_data = [(\"Hello, World!\", param) for param in input_data]\n\nfor variant, (param, result) in enumerate(zip(input_data, result_data), start=1):\n failure_msg=f'Expected: {result} but got something else instead.'\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result, msg=failure_msg)", - "task_id": 2 + "test_code": "input_data = [15, 23, 33, 39]\nresult_data = [(\"Hello, World!\", param) for param in input_data]\n\nfor variant, (param, result) in enumerate(zip(input_data, result_data), start=1):\n failure_msg=f'Expected: {result} but got something else instead.'\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result, msg=failure_msg)", + "task_id": 1 }, { "name": "ExampleAllFailOther > dummy [variation #1]", @@ -170,6 +156,13 @@ "test_code": "input_data = ['frog', 'fish', 'coconut', 'pineapple', 'carrot', 'cucumber', 'grass', 'tree']\nresult_data = [(\"Hello, World!\", param) for param in input_data]\n\nfor variant, (param, result) in enumerate(zip(input_data, result_data), start=1):\n failure_msg=f'Expected: {result} but got something else instead.'\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result, msg=failure_msg)", "task_id": 2 }, + { + "name": "ExampleAllFailOther > dummy~", + "status": "fail", + "message": "One or more variations of this test failed. Details can be found under each [variant#].", + "test_code": "input_data = ['frog', 'fish', 'coconut', 'pineapple', 'carrot', 'cucumber', 'grass', 'tree']\nresult_data = [(\"Hello, World!\", param) for param in input_data]\n\nfor variant, (param, result) in enumerate(zip(input_data, result_data), start=1):\n failure_msg=f'Expected: {result} but got something else instead.'\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result, msg=failure_msg)", + "task_id": 2 + }, { "name": "ExampleAllFailOther > hello [variation #1]", "status": "fail", @@ -197,6 +190,13 @@ "message": "AssertionError: 'Goodbye!' != ('Hello, World!', 10) : Expected: ('Hello, World!', 10) but got something else instead.", "test_code": "input_data = [1, 2, 5, 10]\nresult_data = [(\"Hello, World!\", param) for param in input_data]\n\nfor variant, (param, result) in enumerate(zip(input_data, result_data), start=1):\n failure_msg=f'Expected: {result} but got something else instead.'\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result, msg=failure_msg)", "task_id": 2 + }, + { + "name": "ExampleAllFailOther > hello~", + "status": "fail", + "message": "One or more variations of this test failed. Details can be found under each [variant#].", + "test_code": "input_data = [1, 2, 5, 10]\nresult_data = [(\"Hello, World!\", param) for param in input_data]\n\nfor variant, (param, result) in enumerate(zip(input_data, result_data), start=1):\n failure_msg=f'Expected: {result} but got something else instead.'\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result, msg=failure_msg)", + "task_id": 2 } ] } \ No newline at end of file diff --git a/test/example-config-multiple-files-subtests-and-tasks/results.json b/test/example-config-multiple-files-subtests-and-tasks/results.json index 103a1d7..77a5c2d 100644 --- a/test/example-config-multiple-files-subtests-and-tasks/results.json +++ b/test/example-config-multiple-files-subtests-and-tasks/results.json @@ -3,56 +3,56 @@ "status": "pass", "tests": [ { - "name": "ExampleFirst > abc", + "name": "ExampleFirst > abc~", "status": "pass", "test_code": "self.assertEqual(hello(13), (\"Hello, World!\", 13))", "task_id": 1, "output": "User output is captured! 13" }, { - "name": "ExampleFirst > hello", + "name": "ExampleFirst > hello~", "status": "pass", "test_code": "self.assertEqual(hello('Hi'), (\"Hello, World!\", 'Hi'))", "task_id": 1, "output": "User output is captured! Hi" }, { - "name": "ExampleFirstOther > dummy", + "name": "ExampleFirstOther > dummy~", "status": "pass", "test_code": "self.assertEqual(hello('Banana'), (\"Hello, World!\", \"Banana\"))", "task_id": 2, "output": "User output is captured! Banana" }, { - "name": "ExampleFirstOther > hello", + "name": "ExampleFirstOther > hello~", "status": "pass", "test_code": "self.assertEqual(hello(42), (\"Hello, World!\", 42))", "task_id": 2, "output": "User output is captured! 42" }, { - "name": "ExampleSecond > abc", + "name": "ExampleSecond > abc~", "status": "pass", "test_code": "input_data = ['carrot', 'cucumber', 'grass', 'tree']\nresult_data = [(\"Hello, World!\", param) for param in input_data]\nnumber_of_variants = range(1, len(input_data) + 1)\n\nfor variant, param, result in zip(number_of_variants, input_data, result_data):\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result,\n msg=f'Expected: {result} but got something else instead.')", "task_id": 3, "output": "User output is captured! carrot\nUser output is captured! cucumber\nUser output is captured! grass\nUser output is captured! tree" }, { - "name": "ExampleSecond > hello", + "name": "ExampleSecond > hello~", "status": "pass", "test_code": "input_data = [1, 2, 5]\nresult_data = [(\"Hello, World!\", param) for param in input_data]\nnumber_of_variants = range(1, len(input_data) + 1)\n\nfor variant, param, result in zip(number_of_variants, input_data, result_data):\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result,\n msg=f'Expected: {result} but got something else instead.')", "task_id": 3, "output": "User output is captured! 1\nUser output is captured! 2\nUser output is captured! 5" }, { - "name": "ExampleSecondOther > dummy", + "name": "ExampleSecondOther > dummy~", "status": "pass", "test_code": "input_data = ['frog', 'fish', 'coconut']\nresult_data = [(\"Hello, World!\", param) for param in input_data]\nnumber_of_variants = range(1, len(input_data) + 1)\n\nfor variant, param, result in zip(number_of_variants, input_data, result_data):\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result,\n msg=f'Expected: {result} but got something else instead.')", "task_id": 4, "output": "User output is captured! frog\nUser output is captured! fish\nUser output is captured! coconut" }, { - "name": "ExampleSecondOther > hello", + "name": "ExampleSecondOther > hello~", "status": "pass", "test_code": "input_data = [23, 33, 39]\nresult_data = [(\"Hello, World!\", param) for param in input_data]\nnumber_of_variants = range(1, len(input_data) + 1)\n\nfor variant, param, result in zip(number_of_variants, input_data, result_data):\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result,\n msg=f'Expected: {result} but got something else instead.')", "task_id": 4, diff --git a/test/example-has-stdout-and-tasks/results.json b/test/example-has-stdout-and-tasks/results.json index 31f9c7b..7d77261 100644 --- a/test/example-has-stdout-and-tasks/results.json +++ b/test/example-has-stdout-and-tasks/results.json @@ -3,7 +3,7 @@ "status": "fail", "tests": [ { - "name": "ExampleHasStdoutAndTasks > hello", + "name": "ExampleHasStdoutAndTasks > hello~", "status": "fail", "message": "AssertionError: None != 'Hello, World!'", "test_code": "self.assertEqual(hello(), \"Hello, World!\")", @@ -11,7 +11,7 @@ "output": "Hello, World!" }, { - "name": "ExampleHasStdoutAndTasks > abc", + "name": "ExampleHasStdoutAndTasks > abc~", "status": "fail", "message": "AssertionError: None != 'Hello, World!'", "test_code": "self.assertEqual(hello(), \"Hello, World!\")", @@ -19,7 +19,7 @@ "output": "Hello, World!" }, { - "name": "ExampleHasStdoutAndTasks > truncation", + "name": "ExampleHasStdoutAndTasks > truncation~", "status": "fail", "message": "AssertionError: 'Goodbye!' != 'Hello, World!'\n- Goodbye!\n+ Hello, World!", "test_code": "self.assertEqual(must_truncate(), \"Hello, World!\")", @@ -27,7 +27,7 @@ "output": "Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Vulputate ut pharetra sit amet aliquam. Amet dictum sit amet justo donec enim diam vulputate ut. Consequat nisl vel pretium lectus quam id leo. Maecenas accumsan lacus vel facilisis volutpat est velit egestas dui. Faucibus et molestie ac feugiat sed. Fringilla phasellus faucibus scelerisque eleifend donec pretium vulputate s [Output was truncated. Please limit to 500 chars]" }, { - "name": "ExampleHasStdoutAndTasksOther > dummy", + "name": "ExampleHasStdoutAndTasksOther > dummy~", "status": "fail", "message": "AssertionError: None != 'Hello, World!'", "test_code": "self.assertEqual(hello(), \"Hello, World!\")", @@ -35,7 +35,7 @@ "output": "Hello, World!" }, { - "name": "ExampleHasStdoutAndTasksOther > hello", + "name": "ExampleHasStdoutAndTasksOther > hello~", "status": "fail", "message": "AssertionError: None != 'Hello, World!'", "test_code": "self.assertEqual(hello(), \"Hello, World!\")", diff --git a/test/example-has-stdout-tasks-and-subtests/results.json b/test/example-has-stdout-tasks-and-subtests/results.json index 6cc4651..245b0d9 100644 --- a/test/example-has-stdout-tasks-and-subtests/results.json +++ b/test/example-has-stdout-tasks-and-subtests/results.json @@ -2,13 +2,6 @@ "version": 3, "status": "fail", "tests": [ - { - "name": "ExampleHasStdout > hello", - "status": "fail", - "message": "One or more variations of this test failed. Details can be found under each [variant#].", - "test_code": "input_data = [1, 2, 5, 10, 15, 23, 33, 39]\nresult_data = [(\"Hello, World!\", param) for param in input_data]\nnumber_of_variants = range(1, len(input_data) + 1)\n\nfor variant, param, result in zip(number_of_variants, input_data, result_data):\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result,\n msg=f'Expected: {result} but got something else instead.')", - "task_id": 1 - }, { "name": "ExampleHasStdout > hello [variation #1]", "status": "fail", @@ -74,11 +67,11 @@ "output": "Hello, World! 39" }, { - "name": "ExampleHasStdout > abc", + "name": "ExampleHasStdout > hello~", "status": "fail", "message": "One or more variations of this test failed. Details can be found under each [variant#].", - "test_code": "input_data = ['frog', 'fish', 'coconut', 'pineapple', 'carrot', 'cucumber', 'grass', 'tree']\nresult_data = [(\"Hello, World!\", param) for param in input_data]\nnumber_of_variants = range(1, len(input_data) + 1)\n\nfor variant, param, result in zip(number_of_variants, input_data, result_data):\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result,\n msg=f'Expected: {result} but got something else instead.')", - "task_id": 2 + "test_code": "input_data = [1, 2, 5, 10, 15, 23, 33, 39]\nresult_data = [(\"Hello, World!\", param) for param in input_data]\nnumber_of_variants = range(1, len(input_data) + 1)\n\nfor variant, param, result in zip(number_of_variants, input_data, result_data):\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result,\n msg=f'Expected: {result} but got something else instead.')", + "task_id": 1 }, { "name": "ExampleHasStdout > abc [variation #1]", @@ -145,20 +138,20 @@ "output": "Hello, World! tree" }, { - "name": "ExampleHasStdout > truncation", + "name": "ExampleHasStdout > abc~", + "status": "fail", + "message": "One or more variations of this test failed. Details can be found under each [variant#].", + "test_code": "input_data = ['frog', 'fish', 'coconut', 'pineapple', 'carrot', 'cucumber', 'grass', 'tree']\nresult_data = [(\"Hello, World!\", param) for param in input_data]\nnumber_of_variants = range(1, len(input_data) + 1)\n\nfor variant, param, result in zip(number_of_variants, input_data, result_data):\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result,\n msg=f'Expected: {result} but got something else instead.')", + "task_id": 2 + }, + { + "name": "ExampleHasStdout > truncation~", "status": "fail", "message": "AssertionError: 'Goodbye!' != 'Hello, World!'\n- Goodbye!\n+ Hello, World!", "test_code": "self.assertEqual(must_truncate(), \"Hello, World!\")", "task_id": 3, "output": "Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Vulputate ut pharetra sit amet aliquam. Amet dictum sit amet justo donec enim diam vulputate ut. Consequat nisl vel pretium lectus quam id leo. Maecenas accumsan lacus vel facilisis volutpat est velit egestas dui. Faucibus et molestie ac feugiat sed. Fringilla phasellus faucibus scelerisque eleifend donec pretium vulputate s [Output was truncated. Please limit to 500 chars]" }, - { - "name": "ExampleHasStdoutOther > dummy", - "status": "fail", - "message": "One or more variations of this test failed. Details can be found under each [variant#].", - "test_code": "input_data = ['frog', 'fish', 'coconut', 'pineapple', 'carrot', 'cucumber', 'grass', 'tree']\nresult_data = [(\"Hello, World!\", param) for param in input_data]\nnumber_of_variants = range(1, len(input_data) + 1)\n\nfor variant, param, result in zip(number_of_variants, input_data, result_data):\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result,\n msg=f'Expected: {result} but got something else instead.')", - "task_id": 4 - }, { "name": "ExampleHasStdoutOther > dummy [variation #1]", "status": "fail", @@ -224,11 +217,11 @@ "output": "Hello, World! tree" }, { - "name": "ExampleHasStdoutOther > hello", + "name": "ExampleHasStdoutOther > dummy~", "status": "fail", "message": "One or more variations of this test failed. Details can be found under each [variant#].", - "test_code": "input_data = [1, 2, 5, 10, 15, 23, 33, 39]\nresult_data = [(\"Hello, World!\", param) for param in input_data]\nnumber_of_variants = range(1, len(input_data) + 1)\n\nfor variant, param, result in zip(number_of_variants, input_data, result_data):\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result,\n msg=f'Expected: {result} but got something else instead.')", - "task_id": 5 + "test_code": "input_data = ['frog', 'fish', 'coconut', 'pineapple', 'carrot', 'cucumber', 'grass', 'tree']\nresult_data = [(\"Hello, World!\", param) for param in input_data]\nnumber_of_variants = range(1, len(input_data) + 1)\n\nfor variant, param, result in zip(number_of_variants, input_data, result_data):\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result,\n msg=f'Expected: {result} but got something else instead.')", + "task_id": 4 }, { "name": "ExampleHasStdoutOther > hello [variation #1]", @@ -293,6 +286,13 @@ "test_code": "input_data = [1, 2, 5, 10, 15, 23, 33, 39]\nresult_data = [(\"Hello, World!\", param) for param in input_data]\nnumber_of_variants = range(1, len(input_data) + 1)\n\nfor variant, param, result in zip(number_of_variants, input_data, result_data):\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result,\n msg=f'Expected: {result} but got something else instead.')", "task_id": 5, "output": "Hello, World! 39" + }, + { + "name": "ExampleHasStdoutOther > hello~", + "status": "fail", + "message": "One or more variations of this test failed. Details can be found under each [variant#].", + "test_code": "input_data = [1, 2, 5, 10, 15, 23, 33, 39]\nresult_data = [(\"Hello, World!\", param) for param in input_data]\nnumber_of_variants = range(1, len(input_data) + 1)\n\nfor variant, param, result in zip(number_of_variants, input_data, result_data):\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result,\n msg=f'Expected: {result} but got something else instead.')", + "task_id": 5 } ] } \ No newline at end of file diff --git a/test/example-partial-failure-with-subtests-and-stdout-2/example_partial_failure_with_subtests_and_stdout_2.py b/test/example-partial-failure-with-subtests-and-stdout-2/example_partial_failure_with_subtests_and_stdout_2.py new file mode 100644 index 0000000..6067abd --- /dev/null +++ b/test/example-partial-failure-with-subtests-and-stdout-2/example_partial_failure_with_subtests_and_stdout_2.py @@ -0,0 +1,86 @@ +"""Functions for calculating steps in exchanging currency. + +Python numbers documentation: https://docs.python.org/3/library/stdtypes.html#numeric-types-int-float-complex + +Overview of exchanging currency when travelling: https://www.compareremit.com/money-transfer-tips/guide-to-exchanging-currency-for-overseas-travel/ +""" + + +def exchange_money(budget, exchange_rate): + """ + + :param budget: float - amount of money you are planning to exchange. + :param exchange_rate: float - unit value of the foreign currency. + :return: float - exchanged value of the foreign currency you can receive. + """ + + return budget / exchange_rate + + +def get_change(budget, exchanging_value): + """ + + :param budget: float - amount of money you own. + :param exchanging_value: float - amount of your money you want to exchange now. + :return: float - amount left of your starting currency after exchanging. + """ + + return budget - exchanging_value + + +def get_value_of_bills(denomination, number_of_bills): + """ + + :param denomination: int - the value of a bill. + :param number_of_bills: int - total number of bills. + :return: int - calculated value of the bills. + """ + + return denomination * number_of_bills + + +def get_number_of_bills(amount, denomination): + """ + + :param amount: float - the total starting value. + :param denomination: int - the value of a single bill. + :return: int - number of bills that can be obtained from the amount. + """ + + return int(amount / denomination) + + +def get_leftover_of_bills(amount, denomination): + """ + + :param amount: float - the total starting value. + :param denomination: int - the value of a single bill. + :return: float - the amount that is "leftover", given the current denomination. + """ + + print(f'denomination is:: {denomination}') + print(f'number of bills: {get_number_of_bills(amount, denomination)}') + return amount - denomination * get_number_of_bills(amount, denomination) + + +def exchangeable_value(budget, exchange_rate, spread, denomination): + """ + + :param budget: float - the amount of your money you are planning to exchange. + :param exchange_rate: float - the unit value of the foreign currency. + :param spread: int - percentage that is taken as an exchange fee. + :param denomination: int - the value of a single bill. + :return: int - maximum value you can get. + """ + print(f'params :: budget={budget}, exchange_rate={exchange_rate}, spread={spread}, denomination={denomination}') + + fee = (exchange_rate * spread) / 100 + print('fee is:', fee) + + rate = exchange_rate + fee + print('rate is', rate) + + amount = exchange_money(budget, rate) + print(f'amount is:: {amount}') + + return int(amount / denomination) \ No newline at end of file diff --git a/test/example-partial-failure-with-subtests-and-stdout-2/example_partial_failure_with_subtests_and_stdout_2_test.py b/test/example-partial-failure-with-subtests-and-stdout-2/example_partial_failure_with_subtests_and_stdout_2_test.py new file mode 100644 index 0000000..06adbfb --- /dev/null +++ b/test/example-partial-failure-with-subtests-and-stdout-2/example_partial_failure_with_subtests_and_stdout_2_test.py @@ -0,0 +1,141 @@ +import unittest +import pytest + +from example_partial_failure_with_subtests_and_stdout_2 import ( + exchange_money, + get_change, + get_value_of_bills, + get_number_of_bills, + get_leftover_of_bills, + exchangeable_value) + + +class CurrencyExchangeTest(unittest.TestCase): + @pytest.mark.task(taskno=1) + def test_exchange_money(self): + test_data = [(100000, 0.8), (700000, 10.0)] + result_data = [125000, 70000] + + for variant, (params, expected) in enumerate(zip(test_data, result_data), start=1): + budget, exchange_rate = params + + with self.subTest(f"variation #{variant}", + budget=budget, + exchange_rate=exchange_rate, + expected=expected): + + actual_result = exchange_money(*params) + error_message = (f'Called exchange_money{budget, exchange_rate}. ' + f'The function returned {actual_result}, but ' + f'The tests expected {expected} when exchanging' + f' {budget} at a rate of {exchange_rate}.') + + self.assertAlmostEqual(actual_result, expected, msg=error_message) + + @pytest.mark.task(taskno=2) + def test_get_change(self): + test_data = [(463000, 5000), (1250, 120), (15000, 1380)] + result_data = [458000, 1130, 13620] + + for variant, (params, expected) in enumerate(zip(test_data, result_data), start=1): + budget, exchanging_value = params + + with self.subTest(f"variation #{variant}", + budget=budget, + exchanging_value=exchanging_value, + expected=expected): + + actual_result = get_change(*params) + error_message = (f'Called get_change{budget, exchanging_value}. ' + f'The function returned {actual_result}, but ' + f'The tests expected {expected} left in your budget.') + + self.assertAlmostEqual(actual_result, expected, msg=error_message) + + @pytest.mark.task(taskno=3) + def test_get_value_of_bills(self): + test_data = [(10000, 128), (50, 360), (200, 200)] + result_data = [1280000, 18000, 40000] + + for variant, (params, expected) in enumerate(zip(test_data, result_data), start=1): + denomination, number_of_bills = params + + with self.subTest(f"variation #{variant}", + denomination=denomination, + number_of_bills=number_of_bills, + expected=expected): + + actual_result = get_value_of_bills(*params) + error_message = (f'Called get_value_of_bills{denomination, number_of_bills}. ' + f'The function returned {actual_result}, but ' + f'The tests expected {expected} for the bills value.') + + self.assertEqual(actual_result, expected, msg=error_message) + + @pytest.mark.task(taskno=4) + def test_get_number_of_bills(self): + test_data = [(163270, 50000), (54361, 1000)] + result_data = [3, 54] + + for variant, (params, expected) in enumerate(zip(test_data, result_data), start=1): + amount, denomination = params + + with self.subTest(f"variation #{variant}", + amount=amount, + denomination=denomination, + expected=expected): + + actual_result = get_number_of_bills(amount, denomination) + error_message = (f'Called get_number_of_bills{amount, denomination}. ' + f'The function returned {actual_result} bills, but ' + f'The tests expected {expected} bills.') + + self.assertEqual(actual_result, expected, msg=error_message) + + @pytest.mark.task(taskno=5) + def test_get_leftover_of_bills(self): + test_data = [(10.1, 10), (654321.0, 5), (3.14, 2)] + result_data = [0.1, 1.0, 1.14] + + for variant, (params, expected) in enumerate(zip(test_data, result_data), start=1): + amount, denomination = params + + with self.subTest(f"variation #{variant}", + amount=amount, + denomination=denomination, + expected=expected): + + actual_result = get_leftover_of_bills(*params) + error_message = (f'Called get_leftover_of_bills{amount, denomination}. ' + f'The function returned {actual_result}, but ' + f'The tests expected {expected} as the leftover amount.') + + self.assertAlmostEqual(actual_result, expected, msg=error_message) + + @pytest.mark.task(taskno=6) + def test_exchangeable_value(self): + test_data = [(100000, 10.61, 10, 1), + (1500, 0.84, 25, 40), + (470000, 1050, 30, 10000000000), + (470000, 0.00000009, 30, 700), + (425.33, 0.0009, 30, 700)] + + result_data = [8568, 1400, 0, 4017094016600, 363300] + + for variant, (params, expected) in enumerate(zip(test_data, result_data), start=1): + budget, exchange_rate, spread, denomination = params + + with self.subTest(f"variation #{variant}", + budget=budget, + exchange_rate=exchange_rate, + spread=spread, + denomination=denomination, + expected=expected): + + actual_result = exchangeable_value(budget, exchange_rate, spread, denomination) + error_message = (f'Called exchangeable_value{budget, exchange_rate, spread, denomination}. ' + f'The function returned {actual_result}, but ' + f'The tests expected {expected} as the maximum ' + f'value of the new currency .') + + self.assertEqual(actual_result, expected, msg=error_message) \ No newline at end of file diff --git a/test/example-partial-failure-with-subtests-and-stdout-2/results.json b/test/example-partial-failure-with-subtests-and-stdout-2/results.json new file mode 100644 index 0000000..1754274 --- /dev/null +++ b/test/example-partial-failure-with-subtests-and-stdout-2/results.json @@ -0,0 +1,66 @@ +{ + "version": 3, + "status": "fail", + "tests": [ + { + "name": "CurrencyExchange > exchange money~", + "status": "pass", + "test_code": "test_data = [(100000, 0.8), (700000, 10.0)]\nresult_data = [125000, 70000]\n\nfor variant, (params, expected) in enumerate(zip(test_data, result_data), start=1):\n budget, exchange_rate = params\n\n with self.subTest(f\"variation #{variant}\",\n budget=budget,\n exchange_rate=exchange_rate,\n expected=expected):\n\n actual_result = exchange_money(*params)\n error_message = (f'Called exchange_money{budget, exchange_rate}. '\n f'The function returned {actual_result}, but '\n f'The tests expected {expected} when exchanging'\n f' {budget} at a rate of {exchange_rate}.')\n\n self.assertAlmostEqual(actual_result, expected, msg=error_message)", + "task_id": 1 + }, + { + "name": "CurrencyExchange > get change~", + "status": "pass", + "test_code": "test_data = [(463000, 5000), (1250, 120), (15000, 1380)]\nresult_data = [458000, 1130, 13620]\n\nfor variant, (params, expected) in enumerate(zip(test_data, result_data), start=1):\n budget, exchanging_value = params\n\n with self.subTest(f\"variation #{variant}\",\n budget=budget,\n exchanging_value=exchanging_value,\n expected=expected):\n\n actual_result = get_change(*params)\n error_message = (f'Called get_change{budget, exchanging_value}. '\n f'The function returned {actual_result}, but '\n f'The tests expected {expected} left in your budget.')\n\n self.assertAlmostEqual(actual_result, expected, msg=error_message)", + "task_id": 2 + }, + { + "name": "CurrencyExchange > get value of bills~", + "status": "pass", + "test_code": "test_data = [(10000, 128), (50, 360), (200, 200)]\nresult_data = [1280000, 18000, 40000]\n\nfor variant, (params, expected) in enumerate(zip(test_data, result_data), start=1):\n denomination, number_of_bills = params\n\n with self.subTest(f\"variation #{variant}\",\n denomination=denomination,\n number_of_bills=number_of_bills,\n expected=expected):\n\n actual_result = get_value_of_bills(*params)\n error_message = (f'Called get_value_of_bills{denomination, number_of_bills}. '\n f'The function returned {actual_result}, but '\n f'The tests expected {expected} for the bills value.')\n\n self.assertEqual(actual_result, expected, msg=error_message)", + "task_id": 3 + }, + { + "name": "CurrencyExchange > get number of bills~", + "status": "pass", + "test_code": "test_data = [(163270, 50000), (54361, 1000)]\nresult_data = [3, 54]\n\nfor variant, (params, expected) in enumerate(zip(test_data, result_data), start=1):\n amount, denomination = params\n\n with self.subTest(f\"variation #{variant}\",\n amount=amount,\n denomination=denomination,\n expected=expected):\n\n actual_result = get_number_of_bills(amount, denomination)\n error_message = (f'Called get_number_of_bills{amount, denomination}. '\n f'The function returned {actual_result} bills, but '\n f'The tests expected {expected} bills.')\n\n self.assertEqual(actual_result, expected, msg=error_message)", + "task_id": 4 + }, + { + "name": "CurrencyExchange > get leftover of bills~", + "status": "pass", + "test_code": "test_data = [(10.1, 10), (654321.0, 5), (3.14, 2)]\nresult_data = [0.1, 1.0, 1.14]\n\nfor variant, (params, expected) in enumerate(zip(test_data, result_data), start=1):\n amount, denomination = params\n\n with self.subTest(f\"variation #{variant}\",\n amount=amount,\n denomination=denomination,\n expected=expected):\n\n actual_result = get_leftover_of_bills(*params)\n error_message = (f'Called get_leftover_of_bills{amount, denomination}. '\n f'The function returned {actual_result}, but '\n f'The tests expected {expected} as the leftover amount.')\n\n self.assertAlmostEqual(actual_result, expected, msg=error_message)", + "task_id": 5, + "output": "denomination is:: 10\nnumber of bills: 1\ndenomination is:: 5\nnumber of bills: 130864\ndenomination is:: 2\nnumber of bills: 1" + }, + { + "name": "CurrencyExchange > exchangeable value [variation #2]", + "status": "fail", + "message": "AssertionError: 35 != 1400 : Called exchangeable_value(1500, 0.84, 25, 40). The function returned 35, but The tests expected 1400 as the maximum value of the new currency .", + "test_code": "test_data = [(100000, 10.61, 10, 1),\n (1500, 0.84, 25, 40),\n (470000, 1050, 30, 10000000000),\n (470000, 0.00000009, 30, 700),\n (425.33, 0.0009, 30, 700)]\n\nresult_data = [8568, 1400, 0, 4017094016600, 363300]\n\nfor variant, (params, expected) in enumerate(zip(test_data, result_data), start=1):\n budget, exchange_rate, spread, denomination = params\n\n with self.subTest(f\"variation #{variant}\",\n budget=budget,\n exchange_rate=exchange_rate,\n spread=spread,\n denomination=denomination,\n expected=expected):\n\n actual_result = exchangeable_value(budget, exchange_rate, spread, denomination)\n error_message = (f'Called exchangeable_value{budget, exchange_rate, spread, denomination}. '\n f'The function returned {actual_result}, but '\n f'The tests expected {expected} as the maximum '\n f'value of the new currency .')\n\n self.assertEqual(actual_result, expected, msg=error_message)", + "task_id": 6, + "output": "params :: budget=470000, exchange_rate=1050, spread=30, denomination=10000000000\nfee is: 315.0\nrate is 1365.0\namount is:: 344.32234432234435\nparams :: budget=470000, exchange_rate=9e-08, spread=30, denomination=700\nfee is: 2.7e-08\nrate is 1.17e-07\namount is:: 4017094017094.017" + }, + { + "name": "CurrencyExchange > exchangeable value [variation #4]", + "status": "fail", + "message": "AssertionError: 5738705738 != 4017094016600 : Called exchangeable_value(470000, 9e-08, 30, 700). The function returned 5738705738, but The tests expected 4017094016600 as the maximum value of the new currency .", + "test_code": "test_data = [(100000, 10.61, 10, 1),\n (1500, 0.84, 25, 40),\n (470000, 1050, 30, 10000000000),\n (470000, 0.00000009, 30, 700),\n (425.33, 0.0009, 30, 700)]\n\nresult_data = [8568, 1400, 0, 4017094016600, 363300]\n\nfor variant, (params, expected) in enumerate(zip(test_data, result_data), start=1):\n budget, exchange_rate, spread, denomination = params\n\n with self.subTest(f\"variation #{variant}\",\n budget=budget,\n exchange_rate=exchange_rate,\n spread=spread,\n denomination=denomination,\n expected=expected):\n\n actual_result = exchangeable_value(budget, exchange_rate, spread, denomination)\n error_message = (f'Called exchangeable_value{budget, exchange_rate, spread, denomination}. '\n f'The function returned {actual_result}, but '\n f'The tests expected {expected} as the maximum '\n f'value of the new currency .')\n\n self.assertEqual(actual_result, expected, msg=error_message)", + "task_id": 6 + }, + { + "name": "CurrencyExchange > exchangeable value [variation #5]", + "status": "fail", + "message": "AssertionError: 519 != 363300 : Called exchangeable_value(425.33, 0.0009, 30, 700). The function returned 519, but The tests expected 363300 as the maximum value of the new currency .", + "test_code": "test_data = [(100000, 10.61, 10, 1),\n (1500, 0.84, 25, 40),\n (470000, 1050, 30, 10000000000),\n (470000, 0.00000009, 30, 700),\n (425.33, 0.0009, 30, 700)]\n\nresult_data = [8568, 1400, 0, 4017094016600, 363300]\n\nfor variant, (params, expected) in enumerate(zip(test_data, result_data), start=1):\n budget, exchange_rate, spread, denomination = params\n\n with self.subTest(f\"variation #{variant}\",\n budget=budget,\n exchange_rate=exchange_rate,\n spread=spread,\n denomination=denomination,\n expected=expected):\n\n actual_result = exchangeable_value(budget, exchange_rate, spread, denomination)\n error_message = (f'Called exchangeable_value{budget, exchange_rate, spread, denomination}. '\n f'The function returned {actual_result}, but '\n f'The tests expected {expected} as the maximum '\n f'value of the new currency .')\n\n self.assertEqual(actual_result, expected, msg=error_message)", + "task_id": 6 + }, + { + "name": "CurrencyExchange > exchangeable value~", + "status": "fail", + "message": "One or more variations of this test failed. Details can be found under each [variant#].", + "test_code": "test_data = [(100000, 10.61, 10, 1),\n (1500, 0.84, 25, 40),\n (470000, 1050, 30, 10000000000),\n (470000, 0.00000009, 30, 700),\n (425.33, 0.0009, 30, 700)]\n\nresult_data = [8568, 1400, 0, 4017094016600, 363300]\n\nfor variant, (params, expected) in enumerate(zip(test_data, result_data), start=1):\n budget, exchange_rate, spread, denomination = params\n\n with self.subTest(f\"variation #{variant}\",\n budget=budget,\n exchange_rate=exchange_rate,\n spread=spread,\n denomination=denomination,\n expected=expected):\n\n actual_result = exchangeable_value(budget, exchange_rate, spread, denomination)\n error_message = (f'Called exchangeable_value{budget, exchange_rate, spread, denomination}. '\n f'The function returned {actual_result}, but '\n f'The tests expected {expected} as the maximum '\n f'value of the new currency .')\n\n self.assertEqual(actual_result, expected, msg=error_message)", + "task_id": 6 + } + ] +} \ No newline at end of file diff --git a/test/example-partial-failure-with-subtests-and-stdout/results.json b/test/example-partial-failure-with-subtests-and-stdout/results.json index 1150e8f..3d59ed4 100644 --- a/test/example-partial-failure-with-subtests-and-stdout/results.json +++ b/test/example-partial-failure-with-subtests-and-stdout/results.json @@ -3,15 +3,39 @@ "status": "fail", "tests": [ { - "name": "ExamplePartialFailureWithSubtestsAndStdout > is criticality balanced with passes", + "name": "ExamplePartialFailureWithSubtestsAndStdout > is criticality balanced with passes~", "status": "pass", "test_code": "\"\"\"Testing border cases around typical points.\n\nT, n == (800, 500), (625, 800), (500, 1000), etc.\n\nNo output should be generated in the test report here, since\npassing subtests are not reported on.\n\n\"\"\"\n\ntest_data = ((750, 650, True), (799, 501, True), (500, 600, True),\n (1000, 800, False), (800, 500, False), (800, 500.01, False),\n (799.99, 500, False), (500.01, 999.99, False), (625, 800, False),\n (625.99, 800, False), (625.01, 799.99, False), (799.99, 500.01, True),\n (624.99, 799.99, True), (500, 1000, False), (500.01, 1000, False),\n (499.99, 1000, True))\n\nfor variant, data in enumerate(test_data, start=1):\n temp, neutrons_emitted, expected = data\n with self.subTest(f'variation #{variant}', temp=temp, neutrons_emitted=neutrons_emitted, expected=expected):\n\n # pylint: disable=assignment-from-no-return\n actual_result = is_criticality_balanced(temp, neutrons_emitted)\n failure_message = (f'Expected {expected} but calling is_criticality_balanced(temp={temp}, '\n f'neutrons_emitted={neutrons_emitted}) returned {actual_result}.')\n # f'with T={temp} and neutrons={neutrons_emitted}')\n self.assertEqual(actual_result, expected, failure_message)", "task_id": 1 }, { - "name": "ExamplePartialFailureWithSubtestsAndStdout > reactor efficiency with some subtest output", + "name": "ExamplePartialFailureWithSubtestsAndStdout > reactor efficiency with some subtest output [variation #10]", "status": "fail", - "message": "One or more variations of this test failed. Details can be found under each [variant#].", + "message": "AssertionError: None != 'red' : Expected red as a result, but calling reactor_efficiency(voltage=10, current=300, theoretical_max_power=10000) returned None", + "test_code": "\"\"\"Partial failure and output in the test report.\nThis should happen for:\n - variants 5, 6, 7, 8, 10, 11, and 12.\n\nNo output fields should be present for:\n - the parent or for variations 1, 2, 3, 4, 9 or 13.\n\"\"\"\n\nvoltage = 10\ntheoretical_max_power = 10000\n\n# The numbers are chosen so that current == 10 x percentage\ntest_data = ((1000, 'green'), (999, 'green'), (800, 'green'),\n (799, 'orange'), (700, 'orange'), (600, 'orange'),\n (599, 'red'), (560, 'red'), (400, 'red'), (300, 'red'),\n (299, 'black'), (200, 'black'), (0, 'black'))\n\nfor variant, data in enumerate(test_data, start=1):\n current, expected = data\n with self.subTest(f'variation #{variant}', voltage=voltage, current=current,\n theoretical_max_power=theoretical_max_power, expected=expected):\n\n # pylint: disable=assignment-from-no-return\n actual_result = reactor_efficiency(voltage, current, theoretical_max_power)\n failure_message = (f'Expected {expected} as a result, but calling reactor_efficiency(voltage={voltage}, '\n f'current={current}, theoretical_max_power={theoretical_max_power}) '\n f'returned {actual_result} ')\n # f'with voltage={voltage}, current={current}, max_pow={theoretical_max_power}')\n self.assertEqual(actual_result, expected, failure_message)", + "task_id": 2, + "output": "black" + }, + { + "name": "ExamplePartialFailureWithSubtestsAndStdout > reactor efficiency with some subtest output [variation #11]", + "status": "fail", + "message": "AssertionError: None != 'black' : Expected black as a result, but calling reactor_efficiency(voltage=10, current=299, theoretical_max_power=10000) returned None", + "test_code": "\"\"\"Partial failure and output in the test report.\nThis should happen for:\n - variants 5, 6, 7, 8, 10, 11, and 12.\n\nNo output fields should be present for:\n - the parent or for variations 1, 2, 3, 4, 9 or 13.\n\"\"\"\n\nvoltage = 10\ntheoretical_max_power = 10000\n\n# The numbers are chosen so that current == 10 x percentage\ntest_data = ((1000, 'green'), (999, 'green'), (800, 'green'),\n (799, 'orange'), (700, 'orange'), (600, 'orange'),\n (599, 'red'), (560, 'red'), (400, 'red'), (300, 'red'),\n (299, 'black'), (200, 'black'), (0, 'black'))\n\nfor variant, data in enumerate(test_data, start=1):\n current, expected = data\n with self.subTest(f'variation #{variant}', voltage=voltage, current=current,\n theoretical_max_power=theoretical_max_power, expected=expected):\n\n # pylint: disable=assignment-from-no-return\n actual_result = reactor_efficiency(voltage, current, theoretical_max_power)\n failure_message = (f'Expected {expected} as a result, but calling reactor_efficiency(voltage={voltage}, '\n f'current={current}, theoretical_max_power={theoretical_max_power}) '\n f'returned {actual_result} ')\n # f'with voltage={voltage}, current={current}, max_pow={theoretical_max_power}')\n self.assertEqual(actual_result, expected, failure_message)", + "task_id": 2, + "output": "black" + }, + { + "name": "ExamplePartialFailureWithSubtestsAndStdout > reactor efficiency with some subtest output [variation #12]", + "status": "fail", + "message": "AssertionError: None != 'black' : Expected black as a result, but calling reactor_efficiency(voltage=10, current=200, theoretical_max_power=10000) returned None", + "test_code": "\"\"\"Partial failure and output in the test report.\nThis should happen for:\n - variants 5, 6, 7, 8, 10, 11, and 12.\n\nNo output fields should be present for:\n - the parent or for variations 1, 2, 3, 4, 9 or 13.\n\"\"\"\n\nvoltage = 10\ntheoretical_max_power = 10000\n\n# The numbers are chosen so that current == 10 x percentage\ntest_data = ((1000, 'green'), (999, 'green'), (800, 'green'),\n (799, 'orange'), (700, 'orange'), (600, 'orange'),\n (599, 'red'), (560, 'red'), (400, 'red'), (300, 'red'),\n (299, 'black'), (200, 'black'), (0, 'black'))\n\nfor variant, data in enumerate(test_data, start=1):\n current, expected = data\n with self.subTest(f'variation #{variant}', voltage=voltage, current=current,\n theoretical_max_power=theoretical_max_power, expected=expected):\n\n # pylint: disable=assignment-from-no-return\n actual_result = reactor_efficiency(voltage, current, theoretical_max_power)\n failure_message = (f'Expected {expected} as a result, but calling reactor_efficiency(voltage={voltage}, '\n f'current={current}, theoretical_max_power={theoretical_max_power}) '\n f'returned {actual_result} ')\n # f'with voltage={voltage}, current={current}, max_pow={theoretical_max_power}')\n self.assertEqual(actual_result, expected, failure_message)", + "task_id": 2, + "output": "black" + }, + { + "name": "ExamplePartialFailureWithSubtestsAndStdout > reactor efficiency with some subtest output [variation #13]", + "status": "fail", + "message": "AssertionError: None != 'black' : Expected black as a result, but calling reactor_efficiency(voltage=10, current=0, theoretical_max_power=10000) returned None", "test_code": "\"\"\"Partial failure and output in the test report.\nThis should happen for:\n - variants 5, 6, 7, 8, 10, 11, and 12.\n\nNo output fields should be present for:\n - the parent or for variations 1, 2, 3, 4, 9 or 13.\n\"\"\"\n\nvoltage = 10\ntheoretical_max_power = 10000\n\n# The numbers are chosen so that current == 10 x percentage\ntest_data = ((1000, 'green'), (999, 'green'), (800, 'green'),\n (799, 'orange'), (700, 'orange'), (600, 'orange'),\n (599, 'red'), (560, 'red'), (400, 'red'), (300, 'red'),\n (299, 'black'), (200, 'black'), (0, 'black'))\n\nfor variant, data in enumerate(test_data, start=1):\n current, expected = data\n with self.subTest(f'variation #{variant}', voltage=voltage, current=current,\n theoretical_max_power=theoretical_max_power, expected=expected):\n\n # pylint: disable=assignment-from-no-return\n actual_result = reactor_efficiency(voltage, current, theoretical_max_power)\n failure_message = (f'Expected {expected} as a result, but calling reactor_efficiency(voltage={voltage}, '\n f'current={current}, theoretical_max_power={theoretical_max_power}) '\n f'returned {actual_result} ')\n # f'with voltage={voltage}, current={current}, max_pow={theoretical_max_power}')\n self.assertEqual(actual_result, expected, failure_message)", "task_id": 2 }, @@ -83,42 +107,35 @@ "task_id": 2 }, { - "name": "ExamplePartialFailureWithSubtestsAndStdout > reactor efficiency with some subtest output [variation #10]", - "status": "fail", - "message": "AssertionError: None != 'red' : Expected red as a result, but calling reactor_efficiency(voltage=10, current=300, theoretical_max_power=10000) returned None", - "test_code": "\"\"\"Partial failure and output in the test report.\nThis should happen for:\n - variants 5, 6, 7, 8, 10, 11, and 12.\n\nNo output fields should be present for:\n - the parent or for variations 1, 2, 3, 4, 9 or 13.\n\"\"\"\n\nvoltage = 10\ntheoretical_max_power = 10000\n\n# The numbers are chosen so that current == 10 x percentage\ntest_data = ((1000, 'green'), (999, 'green'), (800, 'green'),\n (799, 'orange'), (700, 'orange'), (600, 'orange'),\n (599, 'red'), (560, 'red'), (400, 'red'), (300, 'red'),\n (299, 'black'), (200, 'black'), (0, 'black'))\n\nfor variant, data in enumerate(test_data, start=1):\n current, expected = data\n with self.subTest(f'variation #{variant}', voltage=voltage, current=current,\n theoretical_max_power=theoretical_max_power, expected=expected):\n\n # pylint: disable=assignment-from-no-return\n actual_result = reactor_efficiency(voltage, current, theoretical_max_power)\n failure_message = (f'Expected {expected} as a result, but calling reactor_efficiency(voltage={voltage}, '\n f'current={current}, theoretical_max_power={theoretical_max_power}) '\n f'returned {actual_result} ')\n # f'with voltage={voltage}, current={current}, max_pow={theoretical_max_power}')\n self.assertEqual(actual_result, expected, failure_message)", - "task_id": 2, - "output": "black" - }, - { - "name": "ExamplePartialFailureWithSubtestsAndStdout > reactor efficiency with some subtest output [variation #11]", + "name": "ExamplePartialFailureWithSubtestsAndStdout > reactor efficiency with some subtest output~", "status": "fail", - "message": "AssertionError: None != 'black' : Expected black as a result, but calling reactor_efficiency(voltage=10, current=299, theoretical_max_power=10000) returned None", + "message": "One or more variations of this test failed. Details can be found under each [variant#].", "test_code": "\"\"\"Partial failure and output in the test report.\nThis should happen for:\n - variants 5, 6, 7, 8, 10, 11, and 12.\n\nNo output fields should be present for:\n - the parent or for variations 1, 2, 3, 4, 9 or 13.\n\"\"\"\n\nvoltage = 10\ntheoretical_max_power = 10000\n\n# The numbers are chosen so that current == 10 x percentage\ntest_data = ((1000, 'green'), (999, 'green'), (800, 'green'),\n (799, 'orange'), (700, 'orange'), (600, 'orange'),\n (599, 'red'), (560, 'red'), (400, 'red'), (300, 'red'),\n (299, 'black'), (200, 'black'), (0, 'black'))\n\nfor variant, data in enumerate(test_data, start=1):\n current, expected = data\n with self.subTest(f'variation #{variant}', voltage=voltage, current=current,\n theoretical_max_power=theoretical_max_power, expected=expected):\n\n # pylint: disable=assignment-from-no-return\n actual_result = reactor_efficiency(voltage, current, theoretical_max_power)\n failure_message = (f'Expected {expected} as a result, but calling reactor_efficiency(voltage={voltage}, '\n f'current={current}, theoretical_max_power={theoretical_max_power}) '\n f'returned {actual_result} ')\n # f'with voltage={voltage}, current={current}, max_pow={theoretical_max_power}')\n self.assertEqual(actual_result, expected, failure_message)", - "task_id": 2, - "output": "black" + "task_id": 2 }, { - "name": "ExamplePartialFailureWithSubtestsAndStdout > reactor efficiency with some subtest output [variation #12]", + "name": "ExamplePartialFailureWithSubtestsAndStdout > fail safe with output truncation [variation #10]", "status": "fail", - "message": "AssertionError: None != 'black' : Expected black as a result, but calling reactor_efficiency(voltage=10, current=200, theoretical_max_power=10000) returned None", - "test_code": "\"\"\"Partial failure and output in the test report.\nThis should happen for:\n - variants 5, 6, 7, 8, 10, 11, and 12.\n\nNo output fields should be present for:\n - the parent or for variations 1, 2, 3, 4, 9 or 13.\n\"\"\"\n\nvoltage = 10\ntheoretical_max_power = 10000\n\n# The numbers are chosen so that current == 10 x percentage\ntest_data = ((1000, 'green'), (999, 'green'), (800, 'green'),\n (799, 'orange'), (700, 'orange'), (600, 'orange'),\n (599, 'red'), (560, 'red'), (400, 'red'), (300, 'red'),\n (299, 'black'), (200, 'black'), (0, 'black'))\n\nfor variant, data in enumerate(test_data, start=1):\n current, expected = data\n with self.subTest(f'variation #{variant}', voltage=voltage, current=current,\n theoretical_max_power=theoretical_max_power, expected=expected):\n\n # pylint: disable=assignment-from-no-return\n actual_result = reactor_efficiency(voltage, current, theoretical_max_power)\n failure_message = (f'Expected {expected} as a result, but calling reactor_efficiency(voltage={voltage}, '\n f'current={current}, theoretical_max_power={theoretical_max_power}) '\n f'returned {actual_result} ')\n # f'with voltage={voltage}, current={current}, max_pow={theoretical_max_power}')\n self.assertEqual(actual_result, expected, failure_message)", - "task_id": 2, - "output": "black" + "message": "AssertionError: None != 'LOW' : Expected LOW but returned None with T=10, neutrons=400, threshold=10000", + "test_code": "\"\"\"All variations of this should fail and appear in the test report.\n - All variations should have output.\n - All output should have truncation warnings.\n\"\"\"\n\ntemp = 10\nthreshold = 10000\ntest_data = ((399, 'LOW'), (300, 'LOW'), (1, 'LOW'),\n (0, 'LOW'), (901, 'NORMAL'), (1000, 'NORMAL'),\n (1099, 'NORMAL'), (899, 'LOW'), (700, 'LOW'),\n (400, 'LOW'), (1101, 'DANGER'), (1200, 'DANGER'))\n\nfor variant, (neutrons_per_second, expected) in enumerate(test_data, start=1):\n with self.subTest(f'variation #{variant}', temp=temp, neutrons_per_second=neutrons_per_second,\n threshold=threshold, expected=expected):\n\n # pylint: disable=assignment-from-no-return\n actual_result = fail_safe(temp, neutrons_per_second, threshold)\n failure_message = (f'Expected {expected} but returned {actual_result} with T={temp}, '\n f'neutrons={neutrons_per_second}, threshold={threshold}')\n self.assertEqual(actual_result, expected, failure_message)", + "task_id": 3, + "output": "Ouptut Captured!!\nId donec ultrices tincidunt arcu non. Semper feugiat nibh sed pulvinar proin gravida hendrerit. Odio ut sem nulla pharetra. Venenatis urna cursus eget nunc scelerisque viverra mauris in. Suscipit adipiscing bibendum est ultricies integer quis. Vel elit scelerisque mauris pellentesque pulvinar. Quam nulla porttitor massa id neque aliquam vestibulum morbi blandit. Ac felis donec et odio pellentesque diam. Vitae tortor condimentum [Output was truncated. Please limit to 500 chars]" }, { - "name": "ExamplePartialFailureWithSubtestsAndStdout > reactor efficiency with some subtest output [variation #13]", + "name": "ExamplePartialFailureWithSubtestsAndStdout > fail safe with output truncation [variation #11]", "status": "fail", - "message": "AssertionError: None != 'black' : Expected black as a result, but calling reactor_efficiency(voltage=10, current=0, theoretical_max_power=10000) returned None", - "test_code": "\"\"\"Partial failure and output in the test report.\nThis should happen for:\n - variants 5, 6, 7, 8, 10, 11, and 12.\n\nNo output fields should be present for:\n - the parent or for variations 1, 2, 3, 4, 9 or 13.\n\"\"\"\n\nvoltage = 10\ntheoretical_max_power = 10000\n\n# The numbers are chosen so that current == 10 x percentage\ntest_data = ((1000, 'green'), (999, 'green'), (800, 'green'),\n (799, 'orange'), (700, 'orange'), (600, 'orange'),\n (599, 'red'), (560, 'red'), (400, 'red'), (300, 'red'),\n (299, 'black'), (200, 'black'), (0, 'black'))\n\nfor variant, data in enumerate(test_data, start=1):\n current, expected = data\n with self.subTest(f'variation #{variant}', voltage=voltage, current=current,\n theoretical_max_power=theoretical_max_power, expected=expected):\n\n # pylint: disable=assignment-from-no-return\n actual_result = reactor_efficiency(voltage, current, theoretical_max_power)\n failure_message = (f'Expected {expected} as a result, but calling reactor_efficiency(voltage={voltage}, '\n f'current={current}, theoretical_max_power={theoretical_max_power}) '\n f'returned {actual_result} ')\n # f'with voltage={voltage}, current={current}, max_pow={theoretical_max_power}')\n self.assertEqual(actual_result, expected, failure_message)", - "task_id": 2 + "message": "AssertionError: None != 'DANGER' : Expected DANGER but returned None with T=10, neutrons=1101, threshold=10000", + "test_code": "\"\"\"All variations of this should fail and appear in the test report.\n - All variations should have output.\n - All output should have truncation warnings.\n\"\"\"\n\ntemp = 10\nthreshold = 10000\ntest_data = ((399, 'LOW'), (300, 'LOW'), (1, 'LOW'),\n (0, 'LOW'), (901, 'NORMAL'), (1000, 'NORMAL'),\n (1099, 'NORMAL'), (899, 'LOW'), (700, 'LOW'),\n (400, 'LOW'), (1101, 'DANGER'), (1200, 'DANGER'))\n\nfor variant, (neutrons_per_second, expected) in enumerate(test_data, start=1):\n with self.subTest(f'variation #{variant}', temp=temp, neutrons_per_second=neutrons_per_second,\n threshold=threshold, expected=expected):\n\n # pylint: disable=assignment-from-no-return\n actual_result = fail_safe(temp, neutrons_per_second, threshold)\n failure_message = (f'Expected {expected} but returned {actual_result} with T={temp}, '\n f'neutrons={neutrons_per_second}, threshold={threshold}')\n self.assertEqual(actual_result, expected, failure_message)", + "task_id": 3, + "output": "Ouptut Captured!!\nId donec ultrices tincidunt arcu non. Semper feugiat nibh sed pulvinar proin gravida hendrerit. Odio ut sem nulla pharetra. Venenatis urna cursus eget nunc scelerisque viverra mauris in. Suscipit adipiscing bibendum est ultricies integer quis. Vel elit scelerisque mauris pellentesque pulvinar. Quam nulla porttitor massa id neque aliquam vestibulum morbi blandit. Ac felis donec et odio pellentesque diam. Vitae tortor condimentum [Output was truncated. Please limit to 500 chars]" }, { - "name": "ExamplePartialFailureWithSubtestsAndStdout > fail safe with output truncation", + "name": "ExamplePartialFailureWithSubtestsAndStdout > fail safe with output truncation [variation #12]", "status": "fail", - "message": "One or more variations of this test failed. Details can be found under each [variant#].", + "message": "AssertionError: None != 'DANGER' : Expected DANGER but returned None with T=10, neutrons=1200, threshold=10000", "test_code": "\"\"\"All variations of this should fail and appear in the test report.\n - All variations should have output.\n - All output should have truncation warnings.\n\"\"\"\n\ntemp = 10\nthreshold = 10000\ntest_data = ((399, 'LOW'), (300, 'LOW'), (1, 'LOW'),\n (0, 'LOW'), (901, 'NORMAL'), (1000, 'NORMAL'),\n (1099, 'NORMAL'), (899, 'LOW'), (700, 'LOW'),\n (400, 'LOW'), (1101, 'DANGER'), (1200, 'DANGER'))\n\nfor variant, (neutrons_per_second, expected) in enumerate(test_data, start=1):\n with self.subTest(f'variation #{variant}', temp=temp, neutrons_per_second=neutrons_per_second,\n threshold=threshold, expected=expected):\n\n # pylint: disable=assignment-from-no-return\n actual_result = fail_safe(temp, neutrons_per_second, threshold)\n failure_message = (f'Expected {expected} but returned {actual_result} with T={temp}, '\n f'neutrons={neutrons_per_second}, threshold={threshold}')\n self.assertEqual(actual_result, expected, failure_message)", - "task_id": 3 + "task_id": 3, + "output": "Ouptut Captured!!\nId donec ultrices tincidunt arcu non. Semper feugiat nibh sed pulvinar proin gravida hendrerit. Odio ut sem nulla pharetra. Venenatis urna cursus eget nunc scelerisque viverra mauris in. Suscipit adipiscing bibendum est ultricies integer quis. Vel elit scelerisque mauris pellentesque pulvinar. Quam nulla porttitor massa id neque aliquam vestibulum morbi blandit. Ac felis donec et odio pellentesque diam. Vitae tortor condimentum [Output was truncated. Please limit to 500 chars]" }, { "name": "ExamplePartialFailureWithSubtestsAndStdout > fail safe with output truncation [variation #1]", @@ -193,28 +210,11 @@ "output": "Ouptut Captured!!\nId donec ultrices tincidunt arcu non. Semper feugiat nibh sed pulvinar proin gravida hendrerit. Odio ut sem nulla pharetra. Venenatis urna cursus eget nunc scelerisque viverra mauris in. Suscipit adipiscing bibendum est ultricies integer quis. Vel elit scelerisque mauris pellentesque pulvinar. Quam nulla porttitor massa id neque aliquam vestibulum morbi blandit. Ac felis donec et odio pellentesque diam. Vitae tortor condimentum [Output was truncated. Please limit to 500 chars]" }, { - "name": "ExamplePartialFailureWithSubtestsAndStdout > fail safe with output truncation [variation #10]", - "status": "fail", - "message": "AssertionError: None != 'LOW' : Expected LOW but returned None with T=10, neutrons=400, threshold=10000", - "test_code": "\"\"\"All variations of this should fail and appear in the test report.\n - All variations should have output.\n - All output should have truncation warnings.\n\"\"\"\n\ntemp = 10\nthreshold = 10000\ntest_data = ((399, 'LOW'), (300, 'LOW'), (1, 'LOW'),\n (0, 'LOW'), (901, 'NORMAL'), (1000, 'NORMAL'),\n (1099, 'NORMAL'), (899, 'LOW'), (700, 'LOW'),\n (400, 'LOW'), (1101, 'DANGER'), (1200, 'DANGER'))\n\nfor variant, (neutrons_per_second, expected) in enumerate(test_data, start=1):\n with self.subTest(f'variation #{variant}', temp=temp, neutrons_per_second=neutrons_per_second,\n threshold=threshold, expected=expected):\n\n # pylint: disable=assignment-from-no-return\n actual_result = fail_safe(temp, neutrons_per_second, threshold)\n failure_message = (f'Expected {expected} but returned {actual_result} with T={temp}, '\n f'neutrons={neutrons_per_second}, threshold={threshold}')\n self.assertEqual(actual_result, expected, failure_message)", - "task_id": 3, - "output": "Ouptut Captured!!\nId donec ultrices tincidunt arcu non. Semper feugiat nibh sed pulvinar proin gravida hendrerit. Odio ut sem nulla pharetra. Venenatis urna cursus eget nunc scelerisque viverra mauris in. Suscipit adipiscing bibendum est ultricies integer quis. Vel elit scelerisque mauris pellentesque pulvinar. Quam nulla porttitor massa id neque aliquam vestibulum morbi blandit. Ac felis donec et odio pellentesque diam. Vitae tortor condimentum [Output was truncated. Please limit to 500 chars]" - }, - { - "name": "ExamplePartialFailureWithSubtestsAndStdout > fail safe with output truncation [variation #11]", - "status": "fail", - "message": "AssertionError: None != 'DANGER' : Expected DANGER but returned None with T=10, neutrons=1101, threshold=10000", - "test_code": "\"\"\"All variations of this should fail and appear in the test report.\n - All variations should have output.\n - All output should have truncation warnings.\n\"\"\"\n\ntemp = 10\nthreshold = 10000\ntest_data = ((399, 'LOW'), (300, 'LOW'), (1, 'LOW'),\n (0, 'LOW'), (901, 'NORMAL'), (1000, 'NORMAL'),\n (1099, 'NORMAL'), (899, 'LOW'), (700, 'LOW'),\n (400, 'LOW'), (1101, 'DANGER'), (1200, 'DANGER'))\n\nfor variant, (neutrons_per_second, expected) in enumerate(test_data, start=1):\n with self.subTest(f'variation #{variant}', temp=temp, neutrons_per_second=neutrons_per_second,\n threshold=threshold, expected=expected):\n\n # pylint: disable=assignment-from-no-return\n actual_result = fail_safe(temp, neutrons_per_second, threshold)\n failure_message = (f'Expected {expected} but returned {actual_result} with T={temp}, '\n f'neutrons={neutrons_per_second}, threshold={threshold}')\n self.assertEqual(actual_result, expected, failure_message)", - "task_id": 3, - "output": "Ouptut Captured!!\nId donec ultrices tincidunt arcu non. Semper feugiat nibh sed pulvinar proin gravida hendrerit. Odio ut sem nulla pharetra. Venenatis urna cursus eget nunc scelerisque viverra mauris in. Suscipit adipiscing bibendum est ultricies integer quis. Vel elit scelerisque mauris pellentesque pulvinar. Quam nulla porttitor massa id neque aliquam vestibulum morbi blandit. Ac felis donec et odio pellentesque diam. Vitae tortor condimentum [Output was truncated. Please limit to 500 chars]" - }, - { - "name": "ExamplePartialFailureWithSubtestsAndStdout > fail safe with output truncation [variation #12]", + "name": "ExamplePartialFailureWithSubtestsAndStdout > fail safe with output truncation~", "status": "fail", - "message": "AssertionError: None != 'DANGER' : Expected DANGER but returned None with T=10, neutrons=1200, threshold=10000", + "message": "One or more variations of this test failed. Details can be found under each [variant#].", "test_code": "\"\"\"All variations of this should fail and appear in the test report.\n - All variations should have output.\n - All output should have truncation warnings.\n\"\"\"\n\ntemp = 10\nthreshold = 10000\ntest_data = ((399, 'LOW'), (300, 'LOW'), (1, 'LOW'),\n (0, 'LOW'), (901, 'NORMAL'), (1000, 'NORMAL'),\n (1099, 'NORMAL'), (899, 'LOW'), (700, 'LOW'),\n (400, 'LOW'), (1101, 'DANGER'), (1200, 'DANGER'))\n\nfor variant, (neutrons_per_second, expected) in enumerate(test_data, start=1):\n with self.subTest(f'variation #{variant}', temp=temp, neutrons_per_second=neutrons_per_second,\n threshold=threshold, expected=expected):\n\n # pylint: disable=assignment-from-no-return\n actual_result = fail_safe(temp, neutrons_per_second, threshold)\n failure_message = (f'Expected {expected} but returned {actual_result} with T={temp}, '\n f'neutrons={neutrons_per_second}, threshold={threshold}')\n self.assertEqual(actual_result, expected, failure_message)", - "task_id": 3, - "output": "Ouptut Captured!!\nId donec ultrices tincidunt arcu non. Semper feugiat nibh sed pulvinar proin gravida hendrerit. Odio ut sem nulla pharetra. Venenatis urna cursus eget nunc scelerisque viverra mauris in. Suscipit adipiscing bibendum est ultricies integer quis. Vel elit scelerisque mauris pellentesque pulvinar. Quam nulla porttitor massa id neque aliquam vestibulum morbi blandit. Ac felis donec et odio pellentesque diam. Vitae tortor condimentum [Output was truncated. Please limit to 500 chars]" + "task_id": 3 } ] } \ No newline at end of file diff --git a/test/example-partial-failure-with-subtests/results.json b/test/example-partial-failure-with-subtests/results.json index 50baef5..84ab759 100644 --- a/test/example-partial-failure-with-subtests/results.json +++ b/test/example-partial-failure-with-subtests/results.json @@ -3,18 +3,11 @@ "status": "fail", "tests": [ { - "name": "ExamplePartialFailureWithSubtests > abc", + "name": "ExamplePartialFailureWithSubtests > abc~", "status": "pass", "test_code": "input_data = ['frog', 'fish', 'coconut', 'pineapple', 'carrot', 'cucumber', 'grass', 'tree']\nresult_data = [(\"Hello, World!\", param) for param in input_data]\n\nfor variant, (param, result) in enumerate(zip(input_data, result_data), start=1):\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result,\n msg=f'Expected: {result} but got something else instead.')", "task_id": 1 }, - { - "name": "ExamplePartialFailureWithSubtests > hello", - "status": "fail", - "message": "One or more variations of this test failed. Details can be found under each [variant#].", - "test_code": "input_data = [1, 2, 5, 10]\nresult_data = [(\"Hello, World!\", param) for param in input_data]\n\nfor variant, (param, result) in enumerate(zip(input_data, result_data), start=1):\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result,\n msg=f'Expected: {result} but got something else instead.')", - "task_id": 1 - }, { "name": "ExamplePartialFailureWithSubtests > hello [variation #1]", "status": "fail", @@ -44,16 +37,16 @@ "task_id": 1 }, { - "name": "ExamplePartialFailureWithSubtestsOther > dummy", - "status": "pass", - "test_code": "input_data = ['frog', 'fish', 'coconut', 'pineapple', 'carrot', 'cucumber', 'grass', 'tree']\nresult_data = [(\"Hello, World!\", param) for param in input_data]\n\nfor variant, (param, result) in enumerate(zip(input_data, result_data), start=1):\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result,\n msg=f'Expected: {result} but got something else instead.')", - "task_id": 2 - }, - { - "name": "ExamplePartialFailureWithSubtestsOther > hello", + "name": "ExamplePartialFailureWithSubtests > hello~", "status": "fail", "message": "One or more variations of this test failed. Details can be found under each [variant#].", - "test_code": "input_data = [15, 23, 33, 39]\nresult_data = [(\"Hello, World!\", param) for param in input_data]\n\nfor variant, (param, result) in enumerate(zip(input_data, result_data), start=1):\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result,\n msg=f'Expected: {result} but got something else instead.')", + "test_code": "input_data = [1, 2, 5, 10]\nresult_data = [(\"Hello, World!\", param) for param in input_data]\n\nfor variant, (param, result) in enumerate(zip(input_data, result_data), start=1):\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result,\n msg=f'Expected: {result} but got something else instead.')", + "task_id": 1 + }, + { + "name": "ExamplePartialFailureWithSubtestsOther > dummy~", + "status": "pass", + "test_code": "input_data = ['frog', 'fish', 'coconut', 'pineapple', 'carrot', 'cucumber', 'grass', 'tree']\nresult_data = [(\"Hello, World!\", param) for param in input_data]\n\nfor variant, (param, result) in enumerate(zip(input_data, result_data), start=1):\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result,\n msg=f'Expected: {result} but got something else instead.')", "task_id": 2 }, { @@ -83,6 +76,13 @@ "message": "AssertionError: 'Hello, World!' != ('Hello, World!', 39) : Expected: ('Hello, World!', 39) but got something else instead.", "test_code": "input_data = [15, 23, 33, 39]\nresult_data = [(\"Hello, World!\", param) for param in input_data]\n\nfor variant, (param, result) in enumerate(zip(input_data, result_data), start=1):\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result,\n msg=f'Expected: {result} but got something else instead.')", "task_id": 2 + }, + { + "name": "ExamplePartialFailureWithSubtestsOther > hello~", + "status": "fail", + "message": "One or more variations of this test failed. Details can be found under each [variant#].", + "test_code": "input_data = [15, 23, 33, 39]\nresult_data = [(\"Hello, World!\", param) for param in input_data]\n\nfor variant, (param, result) in enumerate(zip(input_data, result_data), start=1):\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result,\n msg=f'Expected: {result} but got something else instead.')", + "task_id": 2 } ] } \ No newline at end of file diff --git a/test/example-success-with-subtests/results.json b/test/example-success-with-subtests/results.json index 8738766..d4571fd 100644 --- a/test/example-success-with-subtests/results.json +++ b/test/example-success-with-subtests/results.json @@ -3,28 +3,28 @@ "status": "pass", "tests": [ { - "name": "ExampleSuccessWithSubtests > abc", + "name": "ExampleSuccessWithSubtests > abc~", "status": "pass", "test_code": "input_data = ['frog', 'fish', 'coconut', 'pineapple', 'carrot', 'cucumber', 'grass', 'tree']\nresult_data = [(\"Hello, World!\", param) for param in input_data]\nnumber_of_variants = range(1, len(input_data) + 1)\n\nfor variant, param, result in zip(number_of_variants, input_data, result_data):\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result,\n msg=f'Expected: {result} but got something else instead.')", "task_id": 1, "output": "User output is captured!\nUser output is captured!\nUser output is captured!\nUser output is captured!\nUser output is captured!\nUser output is captured!\nUser output is captured!\nUser output is captured!" }, { - "name": "ExampleSuccessWithSubtests > hello", + "name": "ExampleSuccessWithSubtests > hello~", "status": "pass", "test_code": "input_data = [1, 2, 5, 10, 15, 23, 33, 39]\nresult_data = [(\"Hello, World!\", param) for param in input_data]\nnumber_of_variants = range(1, len(input_data) + 1)\n\nfor variant, param, result in zip(number_of_variants, input_data, result_data):\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result,\n msg=f'Expected: {result} but got something else instead.')", "task_id": 1, "output": "User output is captured!\nUser output is captured!\nUser output is captured!\nUser output is captured!\nUser output is captured!\nUser output is captured!\nUser output is captured!\nUser output is captured!" }, { - "name": "ExampleSuccessWithSubtestsOther > dummy", + "name": "ExampleSuccessWithSubtestsOther > dummy~", "status": "pass", "test_code": "input_data = ['frog', 'fish', 'coconut', 'pineapple', 'carrot', 'cucumber', 'grass', 'tree']\nresult_data = [(\"Hello, World!\", param) for param in input_data]\nnumber_of_variants = range(1, len(input_data) + 1)\n\nfor variant, param, result in zip(number_of_variants, input_data, result_data):\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result,\n msg=f'Expected: {result} but got something else instead.')", "task_id": 2, "output": "User output is captured!\nUser output is captured!\nUser output is captured!\nUser output is captured!\nUser output is captured!\nUser output is captured!\nUser output is captured!\nUser output is captured!" }, { - "name": "ExampleSuccessWithSubtestsOther > hello", + "name": "ExampleSuccessWithSubtestsOther > hello~", "status": "pass", "test_code": "input_data = [1, 2, 5, 10, 15, 23, 33, 39]\nresult_data = [(\"Hello, World!\", param) for param in input_data]\nnumber_of_variants = range(1, len(input_data) + 1)\n\nfor variant, param, result in zip(number_of_variants, input_data, result_data):\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result,\n msg=f'Expected: {result} but got something else instead.')", "task_id": 2,