From 1cd483ebb9c3a29e6c331252dec38cab1fe6d286 Mon Sep 17 00:00:00 2001 From: Philipp Imhof <52650214+PhilippImhof@users.noreply.github.com> Date: Sun, 10 May 2026 15:15:54 +0200 Subject: [PATCH 1/5] do not leak information in adaptive mode --- renderer.php | 60 ++++++++++++++++++++++++-- tests/behat/adaptivenoleak.feature | 67 ++++++++++++++++++++++++++++++ 2 files changed, 123 insertions(+), 4 deletions(-) create mode 100644 tests/behat/adaptivenoleak.feature diff --git a/renderer.php b/renderer.php index a0b9d071..d7b80217 100644 --- a/renderer.php +++ b/renderer.php @@ -200,7 +200,9 @@ public function get_part_feedback_class_and_symbol( $result->feedbacksymbol = ''; $result->feedbackclass = ''; // ... unless correctness is requested in the display options. - if ($options->correctness) { + // Note that no feedback should be given, if the response has been modified since the last submission, + // i. e. it is just a response that was saved during page navigation. + if ($this->response_is_same_as_submitted($qa, $part) && $options->correctness) { $result->feedbacksymbol = $this->feedback_image($result->fraction); $result->feedbackclass = $this->feedback_class($result->fraction); } @@ -859,6 +861,56 @@ public function part_correct_response($part) { ); } + /** + * Check whether the last response of a question attempt is the same as the last submitted response, i. e. it + * was either submitted (e. g. using the "Check" button) or it was saved during page navigation in a quiz but + * still contains the same answers as the ones from the last regular submission. + * + * @param question_attempt $qa + * @param formulas_part|null $part + * @return bool + */ + protected function response_is_same_as_submitted(question_attempt $qa, formulas_part|null $part = null): bool { + // If the last step contains the behaviour var 'submit', it was itself a submitted response. + $laststep = $qa->get_last_step(); + if ($laststep->has_behaviour_var('submit')) { + return true; + } + + // Otherwise, we try to fetch the step containing the last submitted response. + $lastsubmitted = $qa->get_last_step_with_behaviour_var('submit'); + $lastsubmitteddata = $lastsubmitted->get_qt_data(); + + // If there is no data, then no response has ever been submitted. + if (empty($lastsubmitteddata)) { + return false; + } + + // If we have a part, we compare the last step's data to the one from the last submitted response, + // but only for the fields of the relevant part. + $lastdata = $laststep->get_qt_data(); + if ($part !== null) { + return $part->is_same_response($lastsubmitteddata, $lastdata); + } + + // If we do not have a part, we compare the reponse for the entire question. + /** @var qtype_formulas_question $question */ + $question = $qa->get_question(); + + return $question->is_same_response($lastsubmitteddata, $lastdata); + } + + #[\Override] + public function feedback(question_attempt $qa, question_display_options $options) { + // We should not give feedback if the response is not properly submitted, but rather just saved + // during navigation through the quiz. + if (!$this->response_is_same_as_submitted($qa)) { + return ''; + } + + return parent::feedback($qa, $options); + } + /** * Generate a brief statement of how many sub-parts of this question the * student got right. @@ -979,9 +1031,9 @@ protected function part_general_feedback(question_attempt $qa, question_display_ $gradingdetailsdiv = $renderer->render_adaptive_marks($details, $options); $state = $details->state; } - // If the question is in a state that does not yet allow to give a feedback, - // we return an empty string. - if (empty($state->get_feedback_class())) { + // If the question is in a state that does not yet allow to give a feedback + // or if the response is not the last one to be checked, we return an empty string. + if (!$this->response_is_same_as_submitted($qa, $part) || empty($state->get_feedback_class())) { return ''; } diff --git a/tests/behat/adaptivenoleak.feature b/tests/behat/adaptivenoleak.feature new file mode 100644 index 00000000..48a5866e --- /dev/null +++ b/tests/behat/adaptivenoleak.feature @@ -0,0 +1,67 @@ +@qtype @qtype_formulas +Feature: Make sure we do not leak information in adaptive mode + + Background: + Given the following "users" exist: + | username | + | student | + And the following "courses" exist: + | fullname | shortname | category | + | Course 1 | C1 | 0 | + And the following "course enrolments" exist: + | user | course | role | + | student | C1 | student | + And the following "question categories" exist: + | contextlevel | reference | name | + | Course | C1 | Test questions | + And the following "questions" exist: + | questioncategory | qtype | name | template | + | Test questions | formulas | threeparts | testthreeparts | + And the following "activities" exist: + | activity | name | course | idnumber | preferredbehaviour | + | quiz | Quiz 1 | C1 | quiz1 | adaptive | + And quiz "Quiz 1" contains the following questions: + | question | page | + | threeparts | 1 | + And I log in as "student" + And I am on "Course 1" course homepage + And I follow "Quiz 1" + And I press "Attempt quiz" + + Scenario: Try to leak number of correct parts via navigation alone + When I set the field "Answer for part 1" to "5" + And I set the field "Answer for part 2" to "6" + And I set the field "Answer for part 3" to "7" + And I press "Finish attempt" + And I press "Return to attempt" + Then ".formulaspartfeedback-0" "css_element" should not exist + Then ".formulaspartfeedback-1" "css_element" should not exist + Then ".formulaspartfeedback-2" "css_element" should not exist + And ".numpartscorrect" "css_element" should not exist + + Scenario: Try to leak number of correct parts via partial submission and navigation + When I set the field "Answer for part 1" to "5" + And I press "Check" + Then I should see "Marks for this submission" in the ".formulaspartfeedback-0" "css_element" + And I should see "Part 1 correct feedback." + And I should see "You have correctly answered 1 part of this question." + When I set the field "Answer for part 2" to "6" + When I press "Finish attempt" + And I press "Return to attempt" + Then I should see "Marks for this submission" in the ".formulaspartfeedback-0" "css_element" + And I should see "Part 1 correct feedback." + And ".formulaspartfeedback-1" "css_element" should not exist + And ".numpartscorrect" "css_element" should not exist + + Scenario: Part feedback is not shown if answer has been modified since last check + When I set the field "Answer for part 1" to "5" + And I press "Check" + Then I should see "Marks for this submission" in the ".formulaspartfeedback-0" "css_element" + And I should see "Part 1 correct feedback." + And I should see "You have correctly answered 1 part of this question." + When I set the field "Answer for part 1" to "6" + When I press "Finish attempt" + And I press "Return to attempt" + Then ".formulaspartfeedback-0" "css_element" should not exist + And ".gradingdetails" "css_element" should not exist + And ".numpartscorrect" "css_element" should not exist From 1b760284caa5fb2cd0b4dad5c468c29e33e8674e Mon Sep 17 00:00:00 2001 From: Philipp Imhof <52650214+PhilippImhof@users.noreply.github.com> Date: Sun, 10 May 2026 16:13:26 +0200 Subject: [PATCH 2/5] fix behat test review options --- tests/behat/adaptivenoleak.feature | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/behat/adaptivenoleak.feature b/tests/behat/adaptivenoleak.feature index 48a5866e..0d59b137 100644 --- a/tests/behat/adaptivenoleak.feature +++ b/tests/behat/adaptivenoleak.feature @@ -18,8 +18,8 @@ Feature: Make sure we do not leak information in adaptive mode | questioncategory | qtype | name | template | | Test questions | formulas | threeparts | testthreeparts | And the following "activities" exist: - | activity | name | course | idnumber | preferredbehaviour | - | quiz | Quiz 1 | C1 | quiz1 | adaptive | + | activity | name | course | idnumber | preferredbehaviour | reviewcorrectness | reviewmarks | reviewspecificfeedback | reviewgeneralfeedback | + | quiz | Quiz 1 | C1 | quiz1 | adaptive | 65536 | 65536 | 65536 | 65536 | And quiz "Quiz 1" contains the following questions: | question | page | | threeparts | 1 | From 5a1c960a82a9ed930bb03f1d8b17eb28db30deac Mon Sep 17 00:00:00 2001 From: Philipp Imhof <52650214+PhilippImhof@users.noreply.github.com> Date: Sun, 10 May 2026 16:56:05 +0200 Subject: [PATCH 3/5] bugfix for part combined feedback, fix behat test --- renderer.php | 7 +++-- tests/behat/adaptivenoleak.feature | 44 ++++++++++++++++++++++++------ 2 files changed, 40 insertions(+), 11 deletions(-) diff --git a/renderer.php b/renderer.php index d7b80217..346350e5 100644 --- a/renderer.php +++ b/renderer.php @@ -163,8 +163,11 @@ public function part_formulation_and_controls( $feedback .= $this->part_correct_response($part); } - // Put all feedback into a
with the appropriate CSS class and append it to the output. - $output .= html_writer::nonempty_tag('div', $feedback, ['class' => 'formulaspartoutcome outcome']); + // If the current response is a real submission (or identical to one), we put all feedback into a + //
with the appropriate CSS class and append it to the output. + if ($this->response_is_same_as_submitted($qa, $part)) { + $output .= html_writer::nonempty_tag('div', $feedback, ['class' => 'formulaspartoutcome outcome']); + } return html_writer::tag('div', $output, ['class' => 'formulaspart']); } diff --git a/tests/behat/adaptivenoleak.feature b/tests/behat/adaptivenoleak.feature index 0d59b137..3195fae0 100644 --- a/tests/behat/adaptivenoleak.feature +++ b/tests/behat/adaptivenoleak.feature @@ -18,8 +18,8 @@ Feature: Make sure we do not leak information in adaptive mode | questioncategory | qtype | name | template | | Test questions | formulas | threeparts | testthreeparts | And the following "activities" exist: - | activity | name | course | idnumber | preferredbehaviour | reviewcorrectness | reviewmarks | reviewspecificfeedback | reviewgeneralfeedback | - | quiz | Quiz 1 | C1 | quiz1 | adaptive | 65536 | 65536 | 65536 | 65536 | + | activity | name | course | idnumber | preferredbehaviour | + | quiz | Quiz 1 | C1 | quiz1 | adaptive | And quiz "Quiz 1" contains the following questions: | question | page | | threeparts | 1 | @@ -35,20 +35,22 @@ Feature: Make sure we do not leak information in adaptive mode And I press "Finish attempt" And I press "Return to attempt" Then ".formulaspartfeedback-0" "css_element" should not exist - Then ".formulaspartfeedback-1" "css_element" should not exist - Then ".formulaspartfeedback-2" "css_element" should not exist + And ".formulaspartfeedback-1" "css_element" should not exist + And ".formulaspartfeedback-2" "css_element" should not exist And ".numpartscorrect" "css_element" should not exist + And ".fa-circle-check" "css_element" should not exist Scenario: Try to leak number of correct parts via partial submission and navigation When I set the field "Answer for part 1" to "5" And I press "Check" - Then I should see "Marks for this submission" in the ".formulaspartfeedback-0" "css_element" + Then I should see "Marks for this submission" in the ".formulaspartfeedback-0 .gradingdetails" "css_element" And I should see "Part 1 correct feedback." And I should see "You have correctly answered 1 part of this question." + And ".fa-circle-check" "css_element" should exist When I set the field "Answer for part 2" to "6" - When I press "Finish attempt" + And I press "Finish attempt" And I press "Return to attempt" - Then I should see "Marks for this submission" in the ".formulaspartfeedback-0" "css_element" + Then I should see "Marks for this submission" in the ".formulaspartfeedback-0 .gradingdetails" "css_element" And I should see "Part 1 correct feedback." And ".formulaspartfeedback-1" "css_element" should not exist And ".numpartscorrect" "css_element" should not exist @@ -56,12 +58,36 @@ Feature: Make sure we do not leak information in adaptive mode Scenario: Part feedback is not shown if answer has been modified since last check When I set the field "Answer for part 1" to "5" And I press "Check" - Then I should see "Marks for this submission" in the ".formulaspartfeedback-0" "css_element" + Then I should see "Marks for this submission" in the ".formulaspartfeedback-0 .gradingdetails" "css_element" And I should see "Part 1 correct feedback." And I should see "You have correctly answered 1 part of this question." + And ".fa-circle-check" "css_element" should exist When I set the field "Answer for part 1" to "6" - When I press "Finish attempt" + And I press "Finish attempt" And I press "Return to attempt" Then ".formulaspartfeedback-0" "css_element" should not exist And ".gradingdetails" "css_element" should not exist And ".numpartscorrect" "css_element" should not exist + And ".fa-circle-check" "css_element" should not exist + + Scenario: Part feedback is shown when navigating back with an answer identical to the last one that was checked + When I set the field "Answer for part 1" to "5" + And I press "Check" + Then I should see "Marks for this submission" in the ".formulaspartfeedback-0 .gradingdetails" "css_element" + And I should see "Part 1 correct feedback." + And I should see "You have correctly answered 1 part of this question." + And ".fa-circle-check" "css_element" should exist + When I set the field "Answer for part 1" to "6" + And I press "Finish attempt" + And I press "Return to attempt" + Then ".formulaspartfeedback-0" "css_element" should not exist + And ".gradingdetails" "css_element" should not exist + And ".numpartscorrect" "css_element" should not exist + And ".fa-circle-check" "css_element" should not exist + When I set the field "Answer for part 1" to "5" + And I press "Finish attempt" + And I press "Return to attempt" + Then I should see "Marks for this submission" in the ".formulaspartfeedback-0 .gradingdetails" "css_element" + And I should see "Part 1 correct feedback." + And I should see "You have correctly answered 1 part of this question." + And ".fa-circle-check" "css_element" should exist From 7dc96576af7256993567359e4d85f499a3dbe064 Mon Sep 17 00:00:00 2001 From: Philipp Imhof <52650214+PhilippImhof@users.noreply.github.com> Date: Sun, 10 May 2026 17:22:59 +0200 Subject: [PATCH 4/5] fix missing feedback with deferredfeedback --- renderer.php | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/renderer.php b/renderer.php index 346350e5..4f952d82 100644 --- a/renderer.php +++ b/renderer.php @@ -875,8 +875,9 @@ public function part_correct_response($part) { */ protected function response_is_same_as_submitted(question_attempt $qa, formulas_part|null $part = null): bool { // If the last step contains the behaviour var 'submit', it was itself a submitted response. + // For the deferredfeedback behaviour, the step will contain 'finish' instead of 'submit'. $laststep = $qa->get_last_step(); - if ($laststep->has_behaviour_var('submit')) { + if ($laststep->has_behaviour_var('submit') || $laststep->has_behaviour_var('finish')) { return true; } From a4e5efa66627c5d1ac88f5dc8f0e9b97b66bc524 Mon Sep 17 00:00:00 2001 From: Philipp Imhof <52650214+PhilippImhof@users.noreply.github.com> Date: Tue, 12 May 2026 21:16:27 +0200 Subject: [PATCH 5/5] addition to behat test --- tests/behat/adaptivenoleak.feature | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/tests/behat/adaptivenoleak.feature b/tests/behat/adaptivenoleak.feature index 3195fae0..20ec263b 100644 --- a/tests/behat/adaptivenoleak.feature +++ b/tests/behat/adaptivenoleak.feature @@ -55,7 +55,7 @@ Feature: Make sure we do not leak information in adaptive mode And ".formulaspartfeedback-1" "css_element" should not exist And ".numpartscorrect" "css_element" should not exist - Scenario: Part feedback is not shown if answer has been modified since last check + Scenario: Part feedback is not shown if answer has been modified (from correct to wrong) since last check When I set the field "Answer for part 1" to "5" And I press "Check" Then I should see "Marks for this submission" in the ".formulaspartfeedback-0 .gradingdetails" "css_element" @@ -68,8 +68,25 @@ Feature: Make sure we do not leak information in adaptive mode Then ".formulaspartfeedback-0" "css_element" should not exist And ".gradingdetails" "css_element" should not exist And ".numpartscorrect" "css_element" should not exist + And ".fa-circle-xmark" "css_element" should not exist And ".fa-circle-check" "css_element" should not exist + Scenario: Part feedback is not shown if answer has been modified (from wrong to correct) since last check + When I set the field "Answer for part 1" to "6" + And I press "Check" + Then I should see "Marks for this submission: 0.00" in the ".formulaspartfeedback-0 .gradingdetails" "css_element" + And I should see "Part 1 incorrect feedback." + And I should see "You have correctly answered 0 parts of this question." + And ".fa-circle-xmark" "css_element" should exist + When I set the field "Answer for part 1" to "5" + And I press "Finish attempt" + And I press "Return to attempt" + Then ".formulaspartfeedback-0" "css_element" should not exist + And ".gradingdetails" "css_element" should not exist + And ".numpartscorrect" "css_element" should not exist + And ".fa-circle-check" "css_element" should not exist + And ".fa-circle-xmark" "css_element" should not exist + Scenario: Part feedback is shown when navigating back with an answer identical to the last one that was checked When I set the field "Answer for part 1" to "5" And I press "Check"