From c2e4bbdde9eb81ffa2b933d430a694fee78bf5c7 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 29 Sep 2025 04:19:07 +0000 Subject: [PATCH 001/351] chore(deps): bump actions/setup-node from 4 to 5 Bumps [actions/setup-node](https://github.com/actions/setup-node) from 4 to 5. - [Release notes](https://github.com/actions/setup-node/releases) - [Commits](https://github.com/actions/setup-node/compare/v4...v5) --- updated-dependencies: - dependency-name: actions/setup-node dependency-version: '5' dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/js-tests.yml | 2 +- .github/workflows/quality-checks.yml | 2 +- .github/workflows/static-assets-check.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/js-tests.yml b/.github/workflows/js-tests.yml index 94a1368e96a5..9252a823eb18 100644 --- a/.github/workflows/js-tests.yml +++ b/.github/workflows/js-tests.yml @@ -23,7 +23,7 @@ jobs: run: git fetch --depth=1 origin master - name: Setup Node - uses: actions/setup-node@v4 + uses: actions/setup-node@v5 with: node-version: ${{ matrix.node-version }} cache: 'npm' diff --git a/.github/workflows/quality-checks.yml b/.github/workflows/quality-checks.yml index 3f4cbeeb4df9..964e05b06199 100644 --- a/.github/workflows/quality-checks.yml +++ b/.github/workflows/quality-checks.yml @@ -35,7 +35,7 @@ jobs: python-version: ${{ matrix.python-version }} - name: Setup Node - uses: actions/setup-node@v4 + uses: actions/setup-node@v5 with: node-version: ${{ matrix.node-version }} diff --git a/.github/workflows/static-assets-check.yml b/.github/workflows/static-assets-check.yml index 43cb597c16d7..d78d67d58375 100644 --- a/.github/workflows/static-assets-check.yml +++ b/.github/workflows/static-assets-check.yml @@ -48,7 +48,7 @@ jobs: sudo apt-get install libxmlsec1-dev pkg-config - name: Setup Node - uses: actions/setup-node@v4 + uses: actions/setup-node@v5 with: node-version: ${{ matrix.node-version }} From 761ad42290f09f2e6aa962e453e3483d0aaf13ab Mon Sep 17 00:00:00 2001 From: asajjad2 Date: Mon, 29 Sep 2025 16:00:24 +0500 Subject: [PATCH 002/351] fix: prevent errant body string on title edit --- .../js/discussion/views/discussion_thread_edit_view.js | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/common/static/common/js/discussion/views/discussion_thread_edit_view.js b/common/static/common/js/discussion/views/discussion_thread_edit_view.js index ca1c2043e37e..29378548abc2 100644 --- a/common/static/common/js/discussion/views/discussion_thread_edit_view.js +++ b/common/static/common/js/discussion/views/discussion_thread_edit_view.js @@ -7,7 +7,10 @@ tagName: 'form', events: { submit: 'updateHandler', - 'click .post-cancel': 'cancelHandler' + 'click .post-cancel': 'cancelHandler', + 'keypress input:not(.wmd-input)': function(event) { + return DiscussionUtil.ignoreEnterKey(event); + } }, attributes: { From 0e64cec8f15af7cf80221621ca951a710dff2bb5 Mon Sep 17 00:00:00 2001 From: Diana Huang <2952947+dianakhuang@users.noreply.github.com> Date: Thu, 9 Oct 2025 09:23:36 -0400 Subject: [PATCH 003/351] fix: Update workflows to work on the release-ulmo branch. (#1) --- .../add-depr-ticket-to-depr-board.yml | 19 ------------------ .../workflows/add-remove-label-on-comment.yml | 20 ------------------- .github/workflows/js-tests.yml | 2 +- .github/workflows/lint-imports.yml | 2 +- .github/workflows/lockfileversion-check.yml | 2 +- .github/workflows/migrations-check.yml | 2 +- .github/workflows/quality-checks.yml | 3 +-- .github/workflows/semgrep.yml | 2 +- .github/workflows/shellcheck.yml | 2 +- .github/workflows/static-assets-check.yml | 2 +- .github/workflows/unit-tests.yml | 2 +- .../units-test-scripts-structures-pruning.yml | 2 +- .../units-test-scripts-user-retirement.yml | 2 +- .../upgrade-one-python-dependency.yml | 2 +- .../workflows/upgrade-python-requirements.yml | 2 +- .github/workflows/verify-dunder-init.yml | 2 +- 16 files changed, 14 insertions(+), 54 deletions(-) delete mode 100644 .github/workflows/add-depr-ticket-to-depr-board.yml delete mode 100644 .github/workflows/add-remove-label-on-comment.yml diff --git a/.github/workflows/add-depr-ticket-to-depr-board.yml b/.github/workflows/add-depr-ticket-to-depr-board.yml deleted file mode 100644 index 250e394abc11..000000000000 --- a/.github/workflows/add-depr-ticket-to-depr-board.yml +++ /dev/null @@ -1,19 +0,0 @@ -# Run the workflow that adds new tickets that are either: -# - labelled "DEPR" -# - title starts with "[DEPR]" -# - body starts with "Proposal Date" (this is the first template field) -# to the org-wide DEPR project board - -name: Add newly created DEPR issues to the DEPR project board - -on: - issues: - types: [opened] - -jobs: - routeissue: - uses: openedx/.github/.github/workflows/add-depr-ticket-to-depr-board.yml@master - secrets: - GITHUB_APP_ID: ${{ secrets.GRAPHQL_AUTH_APP_ID }} - GITHUB_APP_PRIVATE_KEY: ${{ secrets.GRAPHQL_AUTH_APP_PEM }} - SLACK_BOT_TOKEN: ${{ secrets.SLACK_ISSUE_BOT_TOKEN }} diff --git a/.github/workflows/add-remove-label-on-comment.yml b/.github/workflows/add-remove-label-on-comment.yml deleted file mode 100644 index 0f369db7d293..000000000000 --- a/.github/workflows/add-remove-label-on-comment.yml +++ /dev/null @@ -1,20 +0,0 @@ -# This workflow runs when a comment is made on the ticket -# If the comment starts with "label: " it tries to apply -# the label indicated in rest of comment. -# If the comment starts with "remove label: ", it tries -# to remove the indicated label. -# Note: Labels are allowed to have spaces and this script does -# not parse spaces (as often a space is legitimate), so the command -# "label: really long lots of words label" will apply the -# label "really long lots of words label" - -name: Allows for the adding and removing of labels via comment - -on: - issue_comment: - types: [created] - -jobs: - add_remove_labels: - uses: openedx/.github/.github/workflows/add-remove-label-on-comment.yml@master - diff --git a/.github/workflows/js-tests.yml b/.github/workflows/js-tests.yml index 94a1368e96a5..972435a820e3 100644 --- a/.github/workflows/js-tests.yml +++ b/.github/workflows/js-tests.yml @@ -4,7 +4,7 @@ on: pull_request: push: branches: - - master + - release-ulmo jobs: run_tests: diff --git a/.github/workflows/lint-imports.yml b/.github/workflows/lint-imports.yml index baf914298be2..17cc4ea0a935 100644 --- a/.github/workflows/lint-imports.yml +++ b/.github/workflows/lint-imports.yml @@ -4,7 +4,7 @@ on: pull_request: push: branches: - - master + - release-ulmo jobs: lint-imports: diff --git a/.github/workflows/lockfileversion-check.yml b/.github/workflows/lockfileversion-check.yml index 736f1f98de13..ed0b7f97de6d 100644 --- a/.github/workflows/lockfileversion-check.yml +++ b/.github/workflows/lockfileversion-check.yml @@ -5,7 +5,7 @@ name: Lockfile Version check on: push: branches: - - master + - release-ulmo pull_request: jobs: diff --git a/.github/workflows/migrations-check.yml b/.github/workflows/migrations-check.yml index cd4d09589c12..686d8d9086b0 100644 --- a/.github/workflows/migrations-check.yml +++ b/.github/workflows/migrations-check.yml @@ -5,7 +5,7 @@ on: pull_request: push: branches: - - master + - release-ulmo jobs: check_migrations: diff --git a/.github/workflows/quality-checks.yml b/.github/workflows/quality-checks.yml index 3f4cbeeb4df9..ee67e3903569 100644 --- a/.github/workflows/quality-checks.yml +++ b/.github/workflows/quality-checks.yml @@ -4,8 +4,7 @@ on: pull_request: push: branches: - - master - - open-release/lilac.master + - release-ulmo jobs: run_tests: diff --git a/.github/workflows/semgrep.yml b/.github/workflows/semgrep.yml index 520cd23a678b..d9d32ab9d36d 100644 --- a/.github/workflows/semgrep.yml +++ b/.github/workflows/semgrep.yml @@ -9,7 +9,7 @@ on: pull_request: push: branches: - - master + - release-ulmo jobs: run_semgrep: diff --git a/.github/workflows/shellcheck.yml b/.github/workflows/shellcheck.yml index 2e5b04bcc2ff..a8df63a20d57 100644 --- a/.github/workflows/shellcheck.yml +++ b/.github/workflows/shellcheck.yml @@ -9,7 +9,7 @@ on: pull_request: push: branches: - - master + - release-ulmo permissions: contents: read diff --git a/.github/workflows/static-assets-check.yml b/.github/workflows/static-assets-check.yml index 43cb597c16d7..3a0afa76deb7 100644 --- a/.github/workflows/static-assets-check.yml +++ b/.github/workflows/static-assets-check.yml @@ -4,7 +4,7 @@ on: pull_request: push: branches: - - master + - release-ulmo jobs: static_assets_check: diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index d8b8c26cd049..b89a1b6eb9f3 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -4,7 +4,7 @@ on: pull_request: push: branches: - - master + - release-ulmo concurrency: # We only need to be running tests for the latest commit on each PR diff --git a/.github/workflows/units-test-scripts-structures-pruning.yml b/.github/workflows/units-test-scripts-structures-pruning.yml index 14a01b592308..ef408cfe66ec 100644 --- a/.github/workflows/units-test-scripts-structures-pruning.yml +++ b/.github/workflows/units-test-scripts-structures-pruning.yml @@ -4,7 +4,7 @@ on: pull_request: push: branches: - - master + - release-ulmo jobs: test: diff --git a/.github/workflows/units-test-scripts-user-retirement.yml b/.github/workflows/units-test-scripts-user-retirement.yml index 889c43a64a48..b43bbf46b0d4 100644 --- a/.github/workflows/units-test-scripts-user-retirement.yml +++ b/.github/workflows/units-test-scripts-user-retirement.yml @@ -4,7 +4,7 @@ on: pull_request: push: branches: - - master + - release-ulmo jobs: test: diff --git a/.github/workflows/upgrade-one-python-dependency.yml b/.github/workflows/upgrade-one-python-dependency.yml index 3f9678593c25..1d8170961865 100644 --- a/.github/workflows/upgrade-one-python-dependency.yml +++ b/.github/workflows/upgrade-one-python-dependency.yml @@ -6,7 +6,7 @@ on: branch: description: "Target branch to create requirements PR against" required: true - default: "master" + default: "release-ulmo" type: string package: description: "Name of package to upgrade" diff --git a/.github/workflows/upgrade-python-requirements.yml b/.github/workflows/upgrade-python-requirements.yml index cbb70b06b79d..90c6be00744c 100644 --- a/.github/workflows/upgrade-python-requirements.yml +++ b/.github/workflows/upgrade-python-requirements.yml @@ -8,7 +8,7 @@ on: branch: description: "Target branch to create requirements PR against" required: true - default: "master" + default: "release-ulmo" jobs: call-upgrade-python-requirements-workflow: # Don't run the weekly upgrade job on forks -- it will send a weekly failure email. diff --git a/.github/workflows/verify-dunder-init.yml b/.github/workflows/verify-dunder-init.yml index c3248def2f33..462f0e06af0b 100644 --- a/.github/workflows/verify-dunder-init.yml +++ b/.github/workflows/verify-dunder-init.yml @@ -3,7 +3,7 @@ name: Verify Dunder __init__.py Files on: pull_request: branches: - - master + - release-ulmo jobs: verify_dunder_init: From 09e86e24b215620f351c532649dacf5db38da8f6 Mon Sep 17 00:00:00 2001 From: Navin Karkera Date: Thu, 9 Oct 2025 22:03:23 +0530 Subject: [PATCH 004/351] refactor!: use String field instead of Dict field to store top_level_downstream_parent_key (#37448) * refactor!: use String field instead of Dict field to store top_level_downstream_parent_key Since this is a new field no production instance should have this field yet. Developers need to delete their old courses as this change will raise error in all course pages. * chore: add `top_level_parent` field in ComponentLink and ContainerLink admin * refactor: use ":" as separator * refactor: block key parsing and tests --- cms/djangoapps/contentstore/admin.py | 2 ++ .../tests/test_downstream_sync_integration.py | 15 +++++------ .../v2/views/tests/test_downstreams.py | 25 +++++++---------- cms/djangoapps/contentstore/tasks.py | 6 +++-- cms/djangoapps/contentstore/utils.py | 11 +++++--- .../xblock_storage_handlers/view_handlers.py | 4 +-- .../xblock_storage_handlers/xblock_helpers.py | 6 ++--- cms/lib/xblock/upstream_sync.py | 8 +++--- xmodule/tests/test_util_keys.py | 27 +++++++++++++++++++ xmodule/util/keys.py | 16 +++++++++-- 10 files changed, 80 insertions(+), 40 deletions(-) diff --git a/cms/djangoapps/contentstore/admin.py b/cms/djangoapps/contentstore/admin.py index 67bb39b7a32a..ac11ea42d0a4 100644 --- a/cms/djangoapps/contentstore/admin.py +++ b/cms/djangoapps/contentstore/admin.py @@ -100,6 +100,7 @@ class ComponentLinkAdmin(admin.ModelAdmin): "upstream_context_key", "downstream_usage_key", "downstream_context_key", + "top_level_parent", "version_synced", "version_declined", "created", @@ -139,6 +140,7 @@ class ContainerLinkAdmin(admin.ModelAdmin): "upstream_context_key", "downstream_usage_key", "downstream_context_key", + "top_level_parent", "version_synced", "version_declined", "created", diff --git a/cms/djangoapps/contentstore/rest_api/v2/views/tests/test_downstream_sync_integration.py b/cms/djangoapps/contentstore/rest_api/v2/views/tests/test_downstream_sync_integration.py index e5cd063e8181..78730fe6a0dc 100644 --- a/cms/djangoapps/contentstore/rest_api/v2/views/tests/test_downstream_sync_integration.py +++ b/cms/djangoapps/contentstore/rest_api/v2/views/tests/test_downstream_sync_integration.py @@ -11,11 +11,10 @@ from freezegun import freeze_time from openedx.core.djangoapps.content_libraries.tests import ContentLibrariesRestApiTest -from cms.djangoapps.contentstore.xblock_storage_handlers.xblock_helpers import get_block_key_dict +from cms.djangoapps.contentstore.xblock_storage_handlers.xblock_helpers import get_block_key_string from xmodule.modulestore.django import modulestore from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase from xmodule.modulestore.tests.factories import BlockFactory, CourseFactory -from xmodule.xml_block import serialize_field @ddt.ddt @@ -296,9 +295,9 @@ def test_unit_sync(self): parent_usage_key=str(self.course_subsection.usage_key), upstream_key=self.upstream_unit["id"], ) - downstream_unit_block_key = serialize_field(get_block_key_dict( + downstream_unit_block_key = get_block_key_string( UsageKey.from_string(downstream_unit["locator"]), - )).replace('"', '"') + ) status = self._get_sync_status(downstream_unit["locator"]) self.assertDictContainsEntries(status, { 'upstream_ref': self.upstream_unit["id"], # e.g. 'lct:CL-TEST:testlib:unit:u1' @@ -898,9 +897,9 @@ def test_unit_sync_with_modified_downstream(self): parent_usage_key=str(self.course_subsection.usage_key), upstream_key=self.upstream_unit["id"], ) - downstream_unit_block_key = serialize_field(get_block_key_dict( + downstream_unit_block_key = get_block_key_string( UsageKey.from_string(downstream_unit["locator"]), - )).replace('"', '"') + ) status = self._get_sync_status(downstream_unit["locator"]) self.assertDictContainsEntries(status, { 'upstream_ref': self.upstream_unit["id"], # e.g. 'lct:CL-TEST:testlib:unit:u1' @@ -1259,9 +1258,9 @@ def test_unit_decline_sync(self): parent_usage_key=str(self.course_subsection.usage_key), upstream_key=self.upstream_unit["id"], ) - downstream_unit_block_key = serialize_field(get_block_key_dict( + downstream_unit_block_key = get_block_key_string( UsageKey.from_string(downstream_unit["locator"]), - )).replace('"', '"') + ) children_downstream_keys = self._get_course_block_children(downstream_unit["locator"]) downstream_problem1 = children_downstream_keys[1] assert "type@problem" in downstream_problem1 diff --git a/cms/djangoapps/contentstore/rest_api/v2/views/tests/test_downstreams.py b/cms/djangoapps/contentstore/rest_api/v2/views/tests/test_downstreams.py index b8f2c8f41057..c6f24496f241 100644 --- a/cms/djangoapps/contentstore/rest_api/v2/views/tests/test_downstreams.py +++ b/cms/djangoapps/contentstore/rest_api/v2/views/tests/test_downstreams.py @@ -16,7 +16,7 @@ from cms.djangoapps.contentstore.helpers import StaticFileNotices from cms.djangoapps.contentstore.tests.utils import CourseTestCase from cms.djangoapps.contentstore.xblock_storage_handlers import view_handlers as xblock_view_handlers -from cms.djangoapps.contentstore.xblock_storage_handlers.xblock_helpers import get_block_key_dict +from cms.djangoapps.contentstore.xblock_storage_handlers.xblock_helpers import get_block_key_string from cms.lib.xblock.upstream_sync import BadUpstream, UpstreamLink from common.djangoapps.student.auth import add_users from common.djangoapps.student.roles import CourseStaffRole @@ -157,7 +157,7 @@ def setUp(self): parent=self.top_level_downstream_unit, upstream=self.html_lib_id_2, upstream_version=1, - top_level_downstream_parent_key=get_block_key_dict( + top_level_downstream_parent_key=get_block_key_string( self.top_level_downstream_unit.usage_key, ) ).usage_key @@ -171,7 +171,7 @@ def setUp(self): parent=self.top_level_downstream_chapter, upstream=self.top_level_subsection_id, upstream_version=1, - top_level_downstream_parent_key=get_block_key_dict( + top_level_downstream_parent_key=get_block_key_string( self.top_level_downstream_chapter.usage_key, ), ) @@ -180,7 +180,7 @@ def setUp(self): parent=self.top_level_downstream_sequential, upstream=self.top_level_unit_id_2, upstream_version=1, - top_level_downstream_parent_key=get_block_key_dict( + top_level_downstream_parent_key=get_block_key_string( self.top_level_downstream_chapter.usage_key, ), ) @@ -189,7 +189,7 @@ def setUp(self): parent=self.top_level_downstream_unit_2, upstream=self.video_lib_id_2, upstream_version=1, - top_level_downstream_parent_key=get_block_key_dict( + top_level_downstream_parent_key=get_block_key_string( self.top_level_downstream_chapter.usage_key, ) ).usage_key @@ -455,17 +455,14 @@ def test_unlink_parent_should_update_children_top_level_parent(self): unit = modulestore().get_item(self.top_level_downstream_unit_2.usage_key) # The sequential is the top-level parent for the unit - assert unit.top_level_downstream_parent_key == { - "id": str(self.top_level_downstream_sequential.usage_key.block_id), - "type": str(self.top_level_downstream_sequential.usage_key.block_type), - } + sequential_block_key = get_block_key_string( + self.top_level_downstream_sequential.usage_key + ) + assert unit.top_level_downstream_parent_key == sequential_block_key video = modulestore().get_item(self.top_level_downstream_video_key) # The sequential is the top-level parent for the video - assert video.top_level_downstream_parent_key == { - "id": str(self.top_level_downstream_sequential.usage_key.block_id), - "type": str(self.top_level_downstream_sequential.usage_key.block_type), - } + assert video.top_level_downstream_parent_key == sequential_block_key all_downstreams = self.client.get( "/api/contentstore/v2/downstreams/", @@ -1249,8 +1246,6 @@ def test_200_get_ready_to_sync_top_level_parents_with_components(self): 'downstream_is_modified': False, }, ] - print(data["results"]) - print(expected) self.assertListEqual(data["results"], expected) def test_200_get_ready_to_sync_top_level_parents_with_containers(self): diff --git a/cms/djangoapps/contentstore/tasks.py b/cms/djangoapps/contentstore/tasks.py index 71b86acca201..91b49b8a37c7 100644 --- a/cms/djangoapps/contentstore/tasks.py +++ b/cms/djangoapps/contentstore/tasks.py @@ -92,6 +92,7 @@ from xmodule.modulestore.xml_exporter import export_course_to_xml, export_library_to_xml from xmodule.modulestore.xml_importer import CourseImportException, import_course_from_xml, import_library_from_xml from xmodule.tabs import StaticTab +from xmodule.util.keys import BlockKey from .models import ComponentLink, ContainerLink, LearningContextLinksStatus, LearningContextLinksStatusChoices from .outlines import update_outline_from_modulestore @@ -1649,10 +1650,11 @@ def handle_create_xblock_upstream_link(usage_key): if not xblock.upstream or not xblock.upstream_version: return if xblock.top_level_downstream_parent_key is not None: + block_key = BlockKey.from_string(xblock.top_level_downstream_parent_key) top_level_parent_usage_key = BlockUsageLocator( xblock.course_id, - xblock.top_level_downstream_parent_key.get('type'), - xblock.top_level_downstream_parent_key.get('id'), + block_key.type, + block_key.id, ) try: ContainerLink.get_by_downstream_usage_key(top_level_parent_usage_key) diff --git a/cms/djangoapps/contentstore/utils.py b/cms/djangoapps/contentstore/utils.py index 3ca1d20bf564..c6861b134dd8 100644 --- a/cms/djangoapps/contentstore/utils.py +++ b/cms/djangoapps/contentstore/utils.py @@ -117,6 +117,7 @@ get_all_partitions_for_course, # lint-amnesty, pylint: disable=wrong-import-order ) from xmodule.services import ConfigurationService, SettingsService, TeamsConfigurationService +from xmodule.util.keys import BlockKey from .models import ComponentLink, ContainerLink @@ -2411,10 +2412,11 @@ def _create_or_update_component_link(created: datetime | None, xblock): top_level_parent_usage_key = None if xblock.top_level_downstream_parent_key is not None: + block_key = BlockKey.from_string(xblock.top_level_downstream_parent_key) top_level_parent_usage_key = BlockUsageLocator( xblock.usage_key.course_key, - xblock.top_level_downstream_parent_key.get('type'), - xblock.top_level_downstream_parent_key.get('id'), + block_key.type, + block_key.id, ) ComponentLink.update_or_create( @@ -2444,10 +2446,11 @@ def _create_or_update_container_link(created: datetime | None, xblock): top_level_parent_usage_key = None if xblock.top_level_downstream_parent_key is not None: + block_key = BlockKey.from_string(xblock.top_level_downstream_parent_key) top_level_parent_usage_key = BlockUsageLocator( xblock.usage_key.course_key, - xblock.top_level_downstream_parent_key.get('type'), - xblock.top_level_downstream_parent_key.get('id'), + block_key.type, + block_key.id, ) ContainerLink.update_or_create( diff --git a/cms/djangoapps/contentstore/xblock_storage_handlers/view_handlers.py b/cms/djangoapps/contentstore/xblock_storage_handlers/view_handlers.py index ae7492ddbc91..78c393532305 100644 --- a/cms/djangoapps/contentstore/xblock_storage_handlers/view_handlers.py +++ b/cms/djangoapps/contentstore/xblock_storage_handlers/view_handlers.py @@ -33,7 +33,7 @@ from pytz import UTC from xblock.core import XBlock from xblock.fields import Scope -from .xblock_helpers import get_block_key_dict +from .xblock_helpers import get_block_key_string from cms.djangoapps.contentstore.config.waffle import SHOW_REVIEW_RULES_FLAG from cms.djangoapps.contentstore.helpers import StaticFileNotices @@ -602,7 +602,7 @@ def sync_library_content( block_id=f"{block_type}{uuid4().hex[:8]}", fields={ "upstream": upstream_key, - "top_level_downstream_parent_key": get_block_key_dict( + "top_level_downstream_parent_key": get_block_key_string( top_level_downstream_parent.usage_key, ), }, diff --git a/cms/djangoapps/contentstore/xblock_storage_handlers/xblock_helpers.py b/cms/djangoapps/contentstore/xblock_storage_handlers/xblock_helpers.py index 322bf530ab84..82ed7297d5af 100644 --- a/cms/djangoapps/contentstore/xblock_storage_handlers/xblock_helpers.py +++ b/cms/djangoapps/contentstore/xblock_storage_handlers/xblock_helpers.py @@ -18,11 +18,11 @@ def usage_key_with_run(usage_key_string: str) -> UsageKey: return usage_key -def get_block_key_dict(usage_key: UsageKey) -> dict: +def get_block_key_string(usage_key: UsageKey) -> str: """ - Converts the usage_key in a dict with the form: `{"type": block_type, "id": block_id}` + Extract block key from UsageKey in string format: `html:my-id`. """ - return BlockKey.from_usage_key(usage_key)._asdict() + return str(BlockKey.from_usage_key(usage_key)) def get_tags_count(xblock: XBlock, include_children=False) -> dict[str, int]: diff --git a/cms/lib/xblock/upstream_sync.py b/cms/lib/xblock/upstream_sync.py index da9a60b422bc..5e812f7035f5 100644 --- a/cms/lib/xblock/upstream_sync.py +++ b/cms/lib/xblock/upstream_sync.py @@ -26,7 +26,7 @@ from opaque_keys.edx.locator import LibraryContainerLocator, LibraryUsageLocatorV2 from opaque_keys.edx.keys import UsageKey from xblock.exceptions import XBlockNotFoundError -from xblock.fields import Scope, String, Integer, Dict, List +from xblock.fields import Scope, String, Integer, List from xblock.core import XBlockMixin, XBlock from xmodule.util.keys import BlockKey @@ -337,7 +337,7 @@ def decline_sync(downstream: XBlock, user_id=None) -> None: def _update_children_top_level_parent( downstream: XBlock, - new_top_level_parent_key: dict[str, str] | None + new_top_level_parent_key: str | None, ) -> list[XBlock]: """ Given a new top-level parent block, update the `top_level_downstream_parent_key` field on the downstream block @@ -357,7 +357,7 @@ def _update_children_top_level_parent( # If the `new_top_level_parent_key` is None, the current level assume the top-level # parent key for its children. child_top_level_parent_key = new_top_level_parent_key if new_top_level_parent_key is not None else ( - BlockKey.from_usage_key(child.usage_key)._asdict() + str(BlockKey.from_usage_key(child.usage_key)) ) affected_blocks.extend(_update_children_top_level_parent(child, child_top_level_parent_key)) @@ -466,7 +466,7 @@ class UpstreamSyncMixin(XBlockMixin): default=None, scope=Scope.settings, hidden=True, enforce_type=True, ) - top_level_downstream_parent_key = Dict( + top_level_downstream_parent_key = String( help=( "The block key ('block_type@block_id') of the downstream block that is the top-level parent of " "this block. This is present if the creation of this block is a consequence of " diff --git a/xmodule/tests/test_util_keys.py b/xmodule/tests/test_util_keys.py index dcd16d6b7873..ceda27847830 100644 --- a/xmodule/tests/test_util_keys.py +++ b/xmodule/tests/test_util_keys.py @@ -2,6 +2,7 @@ Tests for xmodule/util/keys.py """ import ddt +import pytest from unittest import TestCase from unittest.mock import Mock @@ -43,3 +44,29 @@ def test_derive_key(self, source, parent, expected): Test that derive_key returns the expected value. """ assert derive_key(source, parent) == expected + + +@ddt.ddt +class TestBlockKeyParsing(TestCase): + """ + Tests for parsing BlockKeys. + """ + + @ddt.data(['chapter:some-id', 'chapter', 'some-id'], ['section:one-more-id', 'section', 'one-more-id']) + @ddt.unpack + def test_block_key_from_string(self, block_key_str, blockType, blockId): + block_key = BlockKey.from_string(block_key_str) + assert block_key.type == blockType + assert block_key.id == blockId + + @ddt.data('chapter:invalid:some-id', 'sectionone-more-id') + def test_block_key_from_string_error(self, block_key_str): + with pytest.raises(ValueError): + BlockKey.from_string(block_key_str) + + @ddt.data( + [BlockKey('chapter', 'some-id'), 'chapter:some-id'], [BlockKey('section', 'one-more-id'), 'section:one-more-id'] + ) + @ddt.unpack + def test_block_key_to_string(self, block_key, block_key_str): + assert str(block_key) == block_key_str diff --git a/xmodule/util/keys.py b/xmodule/util/keys.py index ceb35b269eb5..9570079200cc 100644 --- a/xmodule/util/keys.py +++ b/xmodule/util/keys.py @@ -4,8 +4,7 @@ Consider moving these into opaque-keys if they generalize well. """ import hashlib -from typing import NamedTuple - +from typing import NamedTuple, Self from opaque_keys.edx.keys import UsageKey @@ -28,6 +27,19 @@ class BlockKey(NamedTuple): def from_usage_key(cls, usage_key): return cls(usage_key.block_type, usage_key.block_id) + def __str__(self) -> str: + return f"{self.type}:{self.id}" + + @classmethod + def from_string(cls, s: str) -> Self: + """ + Convert a BlockKey string into a BlockKey object. + """ + parts = s.split(':') + if len(parts) != 2 or not parts[0] or not parts[1]: + raise ValueError(f"Invalid string format for BlockKey: {s}") + return cls(parts[0], parts[1]) + def derive_key(source: UsageKey, dest_parent: BlockKey) -> BlockKey: """ From 272b1669c2c1c70a1b532dfce71ad03ca16b1311 Mon Sep 17 00:00:00 2001 From: Braden MacDonald Date: Thu, 9 Oct 2025 14:31:31 -0700 Subject: [PATCH 005/351] build: enable CI checks in merge queues / merge groups on GitHub --- .github/workflows/check-consistent-dependencies.yml | 1 + .github/workflows/check_python_dependencies.yml | 1 + .github/workflows/ci-static-analysis.yml | 4 +++- .github/workflows/commitlint.yml | 3 ++- .github/workflows/js-tests.yml | 1 + .github/workflows/lint-imports.yml | 1 + .github/workflows/lockfileversion-check.yml | 1 + .github/workflows/migrations-check.yml | 1 + .github/workflows/pylint-checks.yml | 1 + .github/workflows/quality-checks.yml | 1 + .github/workflows/semgrep.yml | 1 + .github/workflows/shellcheck.yml | 1 + .github/workflows/static-assets-check.yml | 1 + .github/workflows/unit-tests.yml | 1 + .github/workflows/units-test-scripts-structures-pruning.yml | 1 + .github/workflows/units-test-scripts-user-retirement.yml | 1 + .github/workflows/verify-dunder-init.yml | 1 + 17 files changed, 20 insertions(+), 2 deletions(-) diff --git a/.github/workflows/check-consistent-dependencies.yml b/.github/workflows/check-consistent-dependencies.yml index c3f35d92a03b..87706e5a09db 100644 --- a/.github/workflows/check-consistent-dependencies.yml +++ b/.github/workflows/check-consistent-dependencies.yml @@ -7,6 +7,7 @@ name: Consistent Python dependencies on: pull_request: + merge_group: defaults: run: diff --git a/.github/workflows/check_python_dependencies.yml b/.github/workflows/check_python_dependencies.yml index 7b93a545cd4b..64da3b985fef 100644 --- a/.github/workflows/check_python_dependencies.yml +++ b/.github/workflows/check_python_dependencies.yml @@ -2,6 +2,7 @@ name: Check Python Dependencies on: pull_request: + merge_group: jobs: check_dependencies: diff --git a/.github/workflows/ci-static-analysis.yml b/.github/workflows/ci-static-analysis.yml index d2513ba2104f..16af7aa69609 100644 --- a/.github/workflows/ci-static-analysis.yml +++ b/.github/workflows/ci-static-analysis.yml @@ -1,6 +1,8 @@ name: Static analysis -on: pull_request +on: + pull_request: + merge_group: jobs: tests: diff --git a/.github/workflows/commitlint.yml b/.github/workflows/commitlint.yml index fec11d6c259b..03b0c6c1336e 100644 --- a/.github/workflows/commitlint.yml +++ b/.github/workflows/commitlint.yml @@ -3,7 +3,8 @@ name: Lint Commit Messages on: - - pull_request + pull_request: + merge_group: jobs: commitlint: diff --git a/.github/workflows/js-tests.yml b/.github/workflows/js-tests.yml index 94a1368e96a5..266e75278315 100644 --- a/.github/workflows/js-tests.yml +++ b/.github/workflows/js-tests.yml @@ -2,6 +2,7 @@ name: Javascript tests on: pull_request: + merge_group: push: branches: - master diff --git a/.github/workflows/lint-imports.yml b/.github/workflows/lint-imports.yml index baf914298be2..a3d2129dd279 100644 --- a/.github/workflows/lint-imports.yml +++ b/.github/workflows/lint-imports.yml @@ -2,6 +2,7 @@ name: Lint Python Imports on: pull_request: + merge_group: push: branches: - master diff --git a/.github/workflows/lockfileversion-check.yml b/.github/workflows/lockfileversion-check.yml index 736f1f98de13..1ebd22b9adaf 100644 --- a/.github/workflows/lockfileversion-check.yml +++ b/.github/workflows/lockfileversion-check.yml @@ -7,6 +7,7 @@ on: branches: - master pull_request: + merge_group: jobs: version-check: diff --git a/.github/workflows/migrations-check.yml b/.github/workflows/migrations-check.yml index cd4d09589c12..966a6635f905 100644 --- a/.github/workflows/migrations-check.yml +++ b/.github/workflows/migrations-check.yml @@ -3,6 +3,7 @@ name: Check Django Migrations on: workflow_dispatch: pull_request: + merge_group: push: branches: - master diff --git a/.github/workflows/pylint-checks.yml b/.github/workflows/pylint-checks.yml index abc51eb91b74..124745cc5ea6 100644 --- a/.github/workflows/pylint-checks.yml +++ b/.github/workflows/pylint-checks.yml @@ -2,6 +2,7 @@ name: Pylint Checks on: pull_request: + merge_group: push: branches: - master diff --git a/.github/workflows/quality-checks.yml b/.github/workflows/quality-checks.yml index 3f4cbeeb4df9..32c576852304 100644 --- a/.github/workflows/quality-checks.yml +++ b/.github/workflows/quality-checks.yml @@ -2,6 +2,7 @@ name: Quality checks on: pull_request: + merge_group: push: branches: - master diff --git a/.github/workflows/semgrep.yml b/.github/workflows/semgrep.yml index 520cd23a678b..8360ae4650d1 100644 --- a/.github/workflows/semgrep.yml +++ b/.github/workflows/semgrep.yml @@ -7,6 +7,7 @@ name: Semgrep code quality on: pull_request: + merge_group: push: branches: - master diff --git a/.github/workflows/shellcheck.yml b/.github/workflows/shellcheck.yml index 2e5b04bcc2ff..ad3d6c3a1e88 100644 --- a/.github/workflows/shellcheck.yml +++ b/.github/workflows/shellcheck.yml @@ -7,6 +7,7 @@ name: ShellCheck on: pull_request: + merge_group: push: branches: - master diff --git a/.github/workflows/static-assets-check.yml b/.github/workflows/static-assets-check.yml index 43cb597c16d7..4c67916431a7 100644 --- a/.github/workflows/static-assets-check.yml +++ b/.github/workflows/static-assets-check.yml @@ -2,6 +2,7 @@ name: static assets check for lms and cms on: pull_request: + merge_group: push: branches: - master diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index 05e5f47d1aae..34a51c5adb9e 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -2,6 +2,7 @@ name: unit-tests on: pull_request: + merge_group: push: branches: - master diff --git a/.github/workflows/units-test-scripts-structures-pruning.yml b/.github/workflows/units-test-scripts-structures-pruning.yml index 14a01b592308..d6575abacbc1 100644 --- a/.github/workflows/units-test-scripts-structures-pruning.yml +++ b/.github/workflows/units-test-scripts-structures-pruning.yml @@ -2,6 +2,7 @@ name: units-test-scripts-common on: pull_request: + merge_group: push: branches: - master diff --git a/.github/workflows/units-test-scripts-user-retirement.yml b/.github/workflows/units-test-scripts-user-retirement.yml index 889c43a64a48..d4051de6bdb4 100644 --- a/.github/workflows/units-test-scripts-user-retirement.yml +++ b/.github/workflows/units-test-scripts-user-retirement.yml @@ -2,6 +2,7 @@ name: units-test-scripts-user-retirement on: pull_request: + merge_group: push: branches: - master diff --git a/.github/workflows/verify-dunder-init.yml b/.github/workflows/verify-dunder-init.yml index c3248def2f33..2a80d40ce510 100644 --- a/.github/workflows/verify-dunder-init.yml +++ b/.github/workflows/verify-dunder-init.yml @@ -4,6 +4,7 @@ on: pull_request: branches: - master + merge_group: jobs: verify_dunder_init: From 0b020a4bf4ed73e72ea97179e56a4538b115ec76 Mon Sep 17 00:00:00 2001 From: asajjad2 Date: Fri, 10 Oct 2025 12:17:29 +0500 Subject: [PATCH 006/351] test: edit form errant behavior --- .../view/discussion_thread_edit_view_spec.js | 43 +++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/common/static/common/js/spec/discussion/view/discussion_thread_edit_view_spec.js b/common/static/common/js/spec/discussion/view/discussion_thread_edit_view_spec.js index 5dbd02b11dea..1aa7b1f4ed4f 100644 --- a/common/static/common/js/spec/discussion/view/discussion_thread_edit_view_spec.js +++ b/common/static/common/js/spec/discussion/view/discussion_thread_edit_view_spec.js @@ -93,6 +93,49 @@ testCancel(this.view); }); + describe('Enter key behavior in title input', function() { + beforeEach(function() { + this.createEditView(); + this.titleInput = this.view.$('.edit-post-title'); + }); + + it('prevents form submission when Enter is pressed in title input', function() { + var submitSpy = jasmine.createSpy('submitSpy'); + this.view.$el.on('submit', submitSpy); + + var enterKeyEvent = $.Event('keypress', {which: 13, keyCode: 13}); + this.titleInput.trigger(enterKeyEvent); + expect(submitSpy).not.toHaveBeenCalled(); + }); + + it('prevents default behavior when Enter is pressed in title input', function() { + var enterKeyEvent = $.Event('keypress', {which: 13, keyCode: 13}); + var preventDefaultSpy = spyOn(enterKeyEvent, 'preventDefault'); + + this.titleInput.trigger(enterKeyEvent); + expect(preventDefaultSpy).toHaveBeenCalled(); + }); + + it('does not prevent non-Enter key presses in title input', function() { + var submitSpy = jasmine.createSpy('submitSpy'); + this.view.$el.on('submit', submitSpy); + + var aKeyEvent = $.Event('keypress', {which: 65, keyCode: 65}); + var preventDefaultSpy = spyOn(aKeyEvent, 'preventDefault'); + this.titleInput.trigger(aKeyEvent); + expect(preventDefaultSpy).not.toHaveBeenCalled(); + }); + + it('does not interfere with body editor when Enter is pressed', function() { + var bodyEditor = this.view.$('.edit-post-body textarea'); + var enterKeyEvent = $.Event('keypress', {which: 13, keyCode: 13}); + var preventDefaultSpy = spyOn(enterKeyEvent, 'preventDefault'); + + bodyEditor.trigger(enterKeyEvent); + expect(preventDefaultSpy).not.toHaveBeenCalled(); + }); + }); + describe('renderComments', function() { beforeEach(function() { this.course_settings = new DiscussionCourseSettings({ From dc074d57cd6ace2c83fff446a63e92c2d1e021b0 Mon Sep 17 00:00:00 2001 From: farhan Date: Fri, 10 Oct 2025 17:25:39 +0500 Subject: [PATCH 007/351] chore: Move sharing_sites into video_configuration app - https://github.com/openedx/edx-platform/issues/37456 --- lms/djangoapps/courseware/tests/test_sharing_sites.py | 4 ++-- .../core/djangoapps/video_config}/sharing_sites.py | 0 xmodule/video_block/video_block.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) rename {xmodule/video_block => openedx/core/djangoapps/video_config}/sharing_sites.py (100%) diff --git a/lms/djangoapps/courseware/tests/test_sharing_sites.py b/lms/djangoapps/courseware/tests/test_sharing_sites.py index 15fcddd584da..0fd0fb9c26e7 100644 --- a/lms/djangoapps/courseware/tests/test_sharing_sites.py +++ b/lms/djangoapps/courseware/tests/test_sharing_sites.py @@ -6,7 +6,7 @@ from unittest import TestCase from unittest.mock import patch from urllib.parse import parse_qsl -from xmodule.video_block.sharing_sites import ( +from openedx.core.djangoapps.video_config.sharing_sites import ( sharing_url, sharing_sites_info_for_video, SharingSiteConfig, @@ -66,7 +66,7 @@ def test_sharing_sites_info_for_video(self): TEST_SHARING_SITE_CONFIG, TEST_SHARING_SITE_CONFIG_WITH_ADDITIONAL_PARAMS, ] - with patch('xmodule.video_block.sharing_sites.ALL_SHARING_SITES', new=sharing_site_configs): + with patch('openedx.core.djangoapps.video_config.sharing_sites.ALL_SHARING_SITES', new=sharing_site_configs): sharing_sites_info = sharing_sites_info_for_video(TEST_PUBLIC_URL, organization=None) for expected_config, actual_info in zip(sharing_site_configs, sharing_sites_info): self.assertDictEqual( diff --git a/xmodule/video_block/sharing_sites.py b/openedx/core/djangoapps/video_config/sharing_sites.py similarity index 100% rename from xmodule/video_block/sharing_sites.py rename to openedx/core/djangoapps/video_config/sharing_sites.py diff --git a/xmodule/video_block/video_block.py b/xmodule/video_block/video_block.py index cc70f812b995..aa0a0b414026 100644 --- a/xmodule/video_block/video_block.py +++ b/xmodule/video_block/video_block.py @@ -58,7 +58,7 @@ ) from xmodule.xml_block import XmlMixin, deserialize_field, is_pointer_tag, name_to_pathname from .bumper_utils import bumperize -from .sharing_sites import sharing_sites_info_for_video +from openedx.core.djangoapps.video_config.sharing_sites import sharing_sites_info_for_video from .transcripts_utils import ( Transcript, VideoTranscriptsMixin, From 6e7b6d866fd2887247a49609f0d7afdfb2ed7e87 Mon Sep 17 00:00:00 2001 From: Feanil Patel Date: Fri, 10 Oct 2025 10:43:37 -0400 Subject: [PATCH 008/351] fix: explicityl set workflows that don't need write access to read-only This came from a github security advisory suggestion but makes sense given that this workflow dosen't need to push content back. Co-authored-by: Copilot Autofix powered by AI <62310815+github-advanced-security[bot]@users.noreply.github.com> --- .github/workflows/unit-tests.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index 05e5f47d1aae..a49bd74f6f6d 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -1,4 +1,6 @@ name: unit-tests +permissions: + contents: read on: pull_request: From 20bc7113e3aa44702680453b7edb80b6ed5038ff Mon Sep 17 00:00:00 2001 From: Kyle McCormick Date: Fri, 10 Oct 2025 12:48:00 -0400 Subject: [PATCH 009/351] feat!: Remove Studio Maintenance & Announcements (#37432) The announcements editor was never ported to frontend-app-authoring, and the announcements display was never ported to frontend-app-learner-dashboard. This announcements feature is rarely used, undocumented, non-a11y-friendly, and there were no volunteers to port it to the new frontends. It is the last remaining part of the legacy Studio "Maintenance" dashboard. So, we are removing it. BREAKING CHANGE: This removes... * Studio Maintenance dashboard legacy frontend * Studio Edit Announcements legacy frontend * The snippet of legacy learner dashboard which renders announcements * openedx/features/announcements djangoapp * The ENABLE_ANNOUNCEMENTS feature flag Not removed: * The announcements_announcement table from openedx/features/announcements . The table is most likely very small, as it is only populated by administrators. Removing it would be more labor for us and more risk of toil for operators than is worthwhile. Closes: https://github.com/openedx/edx-platform/issues/36263 --- .github/workflows/unit-test-shards.json | 1 - cms/djangoapps/maintenance/__init__.py | 0 cms/djangoapps/maintenance/tests.py | 194 ------------------ cms/djangoapps/maintenance/urls.py | 23 --- cms/djangoapps/maintenance/views.py | 190 ----------------- cms/envs/common.py | 2 - cms/static/sass/_build-v1.scss | 1 - cms/static/sass/views/_maintenance.scss | 104 ---------- .../maintenance/_announcement_delete.html | 40 ---- .../maintenance/_announcement_edit.html | 50 ----- .../maintenance/_announcement_index.html | 59 ------ cms/templates/maintenance/base.html | 21 -- cms/templates/maintenance/container.html | 25 --- cms/templates/maintenance/index.html | 20 -- cms/templates/widgets/user_dropdown.html | 5 - cms/urls.py | 2 - lms/static/sass/_build-lms-v1.scss | 1 - lms/static/sass/features/_announcements.scss | 28 --- lms/templates/dashboard.html | 10 - openedx/features/announcements/__init__.py | 0 openedx/features/announcements/apps.py | 32 --- openedx/features/announcements/forms.py | 20 -- .../announcements/migrations/0001_initial.py | 18 -- .../announcements/migrations/__init__.py | 0 openedx/features/announcements/models.py | 22 -- .../announcements/settings/__init__.py | 0 .../features/announcements/settings/common.py | 21 -- .../features/announcements/settings/test.py | 8 - .../announcements/jsx/Announcements.jsx | 141 ------------- .../announcements/jsx/Announcements.test.jsx | 25 --- .../__snapshots__/Announcements.test.jsx.snap | 78 ------- .../announcements/jsx/test-announcements.json | 17 -- .../features/announcements/tests/__init__.py | 0 .../announcements/tests/test_announcements.py | 95 --------- openedx/features/announcements/urls.py | 13 -- openedx/features/announcements/views.py | 37 ---- setup.py | 2 - webpack.common.config.js | 1 - 38 files changed, 1306 deletions(-) delete mode 100644 cms/djangoapps/maintenance/__init__.py delete mode 100644 cms/djangoapps/maintenance/tests.py delete mode 100644 cms/djangoapps/maintenance/urls.py delete mode 100644 cms/djangoapps/maintenance/views.py delete mode 100644 cms/static/sass/views/_maintenance.scss delete mode 100644 cms/templates/maintenance/_announcement_delete.html delete mode 100644 cms/templates/maintenance/_announcement_edit.html delete mode 100644 cms/templates/maintenance/_announcement_index.html delete mode 100644 cms/templates/maintenance/base.html delete mode 100644 cms/templates/maintenance/container.html delete mode 100644 cms/templates/maintenance/index.html delete mode 100644 lms/static/sass/features/_announcements.scss delete mode 100644 openedx/features/announcements/__init__.py delete mode 100644 openedx/features/announcements/apps.py delete mode 100644 openedx/features/announcements/forms.py delete mode 100644 openedx/features/announcements/migrations/0001_initial.py delete mode 100644 openedx/features/announcements/migrations/__init__.py delete mode 100644 openedx/features/announcements/models.py delete mode 100644 openedx/features/announcements/settings/__init__.py delete mode 100644 openedx/features/announcements/settings/common.py delete mode 100644 openedx/features/announcements/settings/test.py delete mode 100644 openedx/features/announcements/static/announcements/jsx/Announcements.jsx delete mode 100644 openedx/features/announcements/static/announcements/jsx/Announcements.test.jsx delete mode 100644 openedx/features/announcements/static/announcements/jsx/__snapshots__/Announcements.test.jsx.snap delete mode 100644 openedx/features/announcements/static/announcements/jsx/test-announcements.json delete mode 100644 openedx/features/announcements/tests/__init__.py delete mode 100644 openedx/features/announcements/tests/test_announcements.py delete mode 100644 openedx/features/announcements/urls.py delete mode 100644 openedx/features/announcements/views.py diff --git a/.github/workflows/unit-test-shards.json b/.github/workflows/unit-test-shards.json index 827366365fa8..cb9beeb3c6bf 100644 --- a/.github/workflows/unit-test-shards.json +++ b/.github/workflows/unit-test-shards.json @@ -239,7 +239,6 @@ "cms/djangoapps/course_creators/", "cms/djangoapps/export_course_metadata/", "cms/djangoapps/modulestore_migrator/", - "cms/djangoapps/maintenance/", "cms/djangoapps/models/", "cms/djangoapps/pipeline_js/", "cms/djangoapps/xblock_config/", diff --git a/cms/djangoapps/maintenance/__init__.py b/cms/djangoapps/maintenance/__init__.py deleted file mode 100644 index e69de29bb2d1..000000000000 diff --git a/cms/djangoapps/maintenance/tests.py b/cms/djangoapps/maintenance/tests.py deleted file mode 100644 index 51f776b00afa..000000000000 --- a/cms/djangoapps/maintenance/tests.py +++ /dev/null @@ -1,194 +0,0 @@ -""" -Tests for the maintenance app views. -""" - - -import ddt -from django.conf import settings -from django.urls import reverse - -from common.djangoapps.student.tests.factories import AdminFactory, UserFactory -from openedx.features.announcements.models import Announcement -from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase # lint-amnesty, pylint: disable=wrong-import-order - -from .views import MAINTENANCE_VIEWS - -# This list contains URLs of all maintenance app views. -MAINTENANCE_URLS = [reverse(view['url']) for view in MAINTENANCE_VIEWS.values()] - - -class TestMaintenanceIndex(ModuleStoreTestCase): - """ - Tests for maintenance index view. - """ - - def setUp(self): - super().setUp() - self.user = AdminFactory() - login_success = self.client.login(username=self.user.username, password=self.TEST_PASSWORD) - self.assertTrue(login_success) - self.view_url = reverse('maintenance:maintenance_index') - - def test_maintenance_index(self): - """ - Test that maintenance index view lists all the maintenance app views. - """ - response = self.client.get(self.view_url) - self.assertContains(response, 'Maintenance', status_code=200) - - # Check that all the expected links appear on the index page. - for url in MAINTENANCE_URLS: - self.assertContains(response, url, status_code=200) - - -@ddt.ddt -class MaintenanceViewTestCase(ModuleStoreTestCase): - """ - Base class for maintenance view tests. - """ - view_url = '' - - def setUp(self): - super().setUp() - self.user = AdminFactory() - login_success = self.client.login(username=self.user.username, password=self.TEST_PASSWORD) - self.assertTrue(login_success) - - def verify_error_message(self, data, error_message): - """ - Verify the response contains error message. - """ - response = self.client.post(self.view_url, data=data, HTTP_X_REQUESTED_WITH='XMLHttpRequest') - self.assertContains(response, error_message, status_code=200) - - def tearDown(self): - """ - Reverse the setup. - """ - self.client.logout() - super().tearDown() - - -@ddt.ddt -class MaintenanceViewAccessTests(MaintenanceViewTestCase): - """ - Tests for access control of maintenance views. - """ - @ddt.data(*MAINTENANCE_URLS) - def test_require_login(self, url): - """ - Test that maintenance app requires user login. - """ - # Log out then try to retrieve the page - self.client.logout() - response = self.client.get(url) - - # Expect a redirect to the login page - redirect_url = '{login_url}?next={original_url}'.format( - login_url=settings.LOGIN_URL, - original_url=url, - ) - - # Studio login redirects to LMS login - self.assertRedirects(response, redirect_url, target_status_code=302) - - @ddt.data(*MAINTENANCE_URLS) - def test_global_staff_access(self, url): - """ - Test that all maintenance app views are accessible to global staff user. - """ - response = self.client.get(url) - self.assertEqual(response.status_code, 200) - - @ddt.data(*MAINTENANCE_URLS) - def test_non_global_staff_access(self, url): - """ - Test that all maintenance app views are not accessible to non-global-staff user. - """ - user = UserFactory(username='test', email='test@example.com', password=self.TEST_PASSWORD) - login_success = self.client.login(username=user.username, password=self.TEST_PASSWORD) - self.assertTrue(login_success) - - response = self.client.get(url) - self.assertContains( - response, - f'Must be {settings.PLATFORM_NAME} staff to perform this action.', - status_code=403 - ) - - -@ddt.ddt -class TestAnnouncementsViews(MaintenanceViewTestCase): - """ - Tests for the announcements edit view. - """ - - def setUp(self): - super().setUp() - self.admin = AdminFactory.create( - email='staff@edx.org', - username='admin', - password=self.TEST_PASSWORD - ) - self.client.login(username=self.admin.username, password=self.TEST_PASSWORD) - self.non_staff_user = UserFactory.create( - email='test@edx.org', - username='test', - password=self.TEST_PASSWORD - ) - - def test_index(self): - """ - Test create announcement view - """ - url = reverse("maintenance:announcement_index") - response = self.client.get(url) - self.assertContains(response, '
') - - def test_create(self): - """ - Test create announcement view - """ - url = reverse("maintenance:announcement_create") - self.client.post(url, {"content": "Test Create Announcement", "active": True}) - result = Announcement.objects.filter(content="Test Create Announcement").exists() - self.assertTrue(result) - - def test_edit(self): - """ - Test edit announcement view - """ - announcement = Announcement.objects.create(content="test") - announcement.save() - url = reverse("maintenance:announcement_edit", kwargs={"pk": announcement.pk}) - response = self.client.get(url) - self.assertContains(response, '
') - self.client.post(url, {"content": "Test Edit Announcement", "active": True}) - announcement = Announcement.objects.get(pk=announcement.pk) - self.assertEqual(announcement.content, "Test Edit Announcement") - - def test_delete(self): - """ - Test delete announcement view - """ - announcement = Announcement.objects.create(content="Test Delete") - announcement.save() - url = reverse("maintenance:announcement_delete", kwargs={"pk": announcement.pk}) - self.client.post(url) - result = Announcement.objects.filter(content="Test Edit Announcement").exists() - self.assertFalse(result) - - def _test_403(self, viewname, kwargs=None): - url = reverse("maintenance:%s" % viewname, kwargs=kwargs) - response = self.client.get(url) - self.assertEqual(response.status_code, 403) - - def test_authorization(self): - self.client.login(username=self.non_staff_user, password=self.TEST_PASSWORD) - announcement = Announcement.objects.create(content="Test Delete") - announcement.save() - - self._test_403("announcement_index") - self._test_403("announcement_create") - self._test_403("announcement_edit", {"pk": announcement.pk}) - self._test_403("announcement_delete", {"pk": announcement.pk}) diff --git a/cms/djangoapps/maintenance/urls.py b/cms/djangoapps/maintenance/urls.py deleted file mode 100644 index 2937727bfa39..000000000000 --- a/cms/djangoapps/maintenance/urls.py +++ /dev/null @@ -1,23 +0,0 @@ -""" -URLs for the maintenance app. -""" - -from django.urls import path, re_path - -from .views import ( - AnnouncementCreateView, - AnnouncementDeleteView, - AnnouncementEditView, - AnnouncementIndexView, - MaintenanceIndexView -) - -app_name = 'cms.djangoapps.maintenance' - -urlpatterns = [ - path('', MaintenanceIndexView.as_view(), name='maintenance_index'), - re_path(r'^announcements/(?P\d+)?$', AnnouncementIndexView.as_view(), name='announcement_index'), - path('announcements/create', AnnouncementCreateView.as_view(), name='announcement_create'), - re_path(r'^announcements/edit/(?P\d+)?$', AnnouncementEditView.as_view(), name='announcement_edit'), - path('announcements/delete/', AnnouncementDeleteView.as_view(), name='announcement_delete'), -] diff --git a/cms/djangoapps/maintenance/views.py b/cms/djangoapps/maintenance/views.py deleted file mode 100644 index 1ec0f9c5cff2..000000000000 --- a/cms/djangoapps/maintenance/views.py +++ /dev/null @@ -1,190 +0,0 @@ -""" -Views for the maintenance app. -""" - - -import logging - -from django.core.validators import ValidationError -from django.urls import reverse, reverse_lazy -from django.utils.decorators import method_decorator -from django.utils.translation import gettext as _ -from django.views.generic import View -from django.views.generic.edit import CreateView, DeleteView, UpdateView -from django.views.generic.list import ListView -from opaque_keys.edx.keys import CourseKey - -from common.djangoapps.edxmako.shortcuts import render_to_response -from common.djangoapps.util.json_request import JsonResponse -from common.djangoapps.util.views import require_global_staff -from openedx.features.announcements.forms import AnnouncementForm -from openedx.features.announcements.models import Announcement -from xmodule.modulestore import ModuleStoreEnum # lint-amnesty, pylint: disable=wrong-import-order -from xmodule.modulestore.django import modulestore # lint-amnesty, pylint: disable=wrong-import-order -from xmodule.modulestore.exceptions import ItemNotFoundError # lint-amnesty, pylint: disable=wrong-import-order - -log = logging.getLogger(__name__) - -# This dict maintains all the views that will be used Maintenance app. -MAINTENANCE_VIEWS = { - 'announcement_index': { - 'url': 'maintenance:announcement_index', - 'name': _('Edit Announcements'), - 'slug': 'announcement_index', - 'description': _( - 'This view shows the announcement editor to create or alter announcements that are shown on the right' - 'side of the dashboard.' - ), - }, -} - - -COURSE_KEY_ERROR_MESSAGES = { - 'empty_course_key': _('Please provide course id.'), - 'invalid_course_key': _('Invalid course key.'), - 'course_key_not_found': _('No matching course found.') -} - - -class MaintenanceIndexView(View): - """ - Index view for maintenance dashboard, used by global staff. - - This view lists some commands/tasks that can be used to dry run or execute directly. - """ - - @method_decorator(require_global_staff) - def get(self, request): - """Render the maintenance index view. """ - return render_to_response('maintenance/index.html', { - 'views': MAINTENANCE_VIEWS, - }) - - -class MaintenanceBaseView(View): - """ - Base class for Maintenance views. - """ - - template = 'maintenance/container.html' - - def __init__(self, view=None): - super().__init__() - self.context = { - 'view': view if view else '', - 'form_data': {}, - 'error': False, - 'msg': '' - } - - def render_response(self): - """ - A short method to render_to_response that renders response. - """ - if self.request.headers.get('x-requested-with') == 'XMLHttpRequest': - return JsonResponse(self.context) - return render_to_response(self.template, self.context) - - @method_decorator(require_global_staff) - def get(self, request): - """ - Render get view. - """ - return self.render_response() - - def validate_course_key(self, course_key, branch=ModuleStoreEnum.BranchName.draft): - """ - Validates the course_key that would be used by maintenance app views. - - Arguments: - course_key (string): a course key - branch: a course locator branch, default value is ModuleStoreEnum.BranchName.draft . - values can be either ModuleStoreEnum.BranchName.draft or ModuleStoreEnum.BranchName.published. - - Returns: - course_usage_key (CourseLocator): course usage locator - """ - if not course_key: - raise ValidationError(COURSE_KEY_ERROR_MESSAGES['empty_course_key']) - - course_usage_key = CourseKey.from_string(course_key) - - if not modulestore().has_course(course_usage_key): - raise ItemNotFoundError(COURSE_KEY_ERROR_MESSAGES['course_key_not_found']) - - # get branch specific locator - course_usage_key = course_usage_key.for_branch(branch) - - return course_usage_key - - -class AnnouncementBaseView(View): - """ - Base view for Announcements pages - """ - - @method_decorator(require_global_staff) - def dispatch(self, request, *args, **kwargs): - return super().dispatch(request, *args, **kwargs) - - -class AnnouncementIndexView(ListView, MaintenanceBaseView): - """ - View for viewing the announcements shown on the dashboard, used by the global staff. - """ - model = Announcement - object_list = Announcement.objects.order_by('-active') - context_object_name = 'announcement_list' - paginate_by = 8 - - def __init__(self): - super().__init__(MAINTENANCE_VIEWS['announcement_index']) - - def get_context_data(self, **kwargs): - context = super().get_context_data(**kwargs) - context['view'] = MAINTENANCE_VIEWS['announcement_index'] - return context - - @method_decorator(require_global_staff) - def get(self, request, *args, **kwargs): - context = self.get_context_data() - return render_to_response(self.template, context) - - -class AnnouncementEditView(UpdateView, AnnouncementBaseView): - """ - View for editing an announcement. - """ - model = Announcement - form_class = AnnouncementForm - success_url = reverse_lazy('maintenance:announcement_index') - template_name = '/maintenance/_announcement_edit.html' - - def get_context_data(self, **kwargs): - context = super().get_context_data(**kwargs) - context['action_url'] = reverse('maintenance:announcement_edit', kwargs={'pk': context['announcement'].pk}) - return context - - -class AnnouncementCreateView(CreateView, AnnouncementBaseView): - """ - View for creating an announcement. - """ - model = Announcement - form_class = AnnouncementForm - success_url = reverse_lazy('maintenance:announcement_index') - template_name = '/maintenance/_announcement_edit.html' - - def get_context_data(self, **kwargs): - context = super().get_context_data(**kwargs) - context['action_url'] = reverse('maintenance:announcement_create') - return context - - -class AnnouncementDeleteView(DeleteView, AnnouncementBaseView): - """ - View for deleting an announcement. - """ - model = Announcement - success_url = reverse_lazy('maintenance:announcement_index') - template_name = '/maintenance/_announcement_delete.html' diff --git a/cms/envs/common.py b/cms/envs/common.py index 99db8900d3c4..e219e3c48daf 100644 --- a/cms/envs/common.py +++ b/cms/envs/common.py @@ -1010,8 +1010,6 @@ # New (Learning-Core-based) XBlock runtime 'openedx.core.djangoapps.xblock.apps.StudioXBlockAppConfig', - # Maintenance tools - 'cms.djangoapps.maintenance', 'openedx.core.djangoapps.util.apps.UtilConfig', # Tracking diff --git a/cms/static/sass/_build-v1.scss b/cms/static/sass/_build-v1.scss index 178f6b167473..7aedd0c6b8a3 100644 --- a/cms/static/sass/_build-v1.scss +++ b/cms/static/sass/_build-v1.scss @@ -77,7 +77,6 @@ @import 'views/group-configuration'; @import 'views/video-upload'; @import 'views/certificates'; -@import 'views/maintenance'; // +Base - Contexts // ==================== diff --git a/cms/static/sass/views/_maintenance.scss b/cms/static/sass/views/_maintenance.scss deleted file mode 100644 index 58d1b7494751..000000000000 --- a/cms/static/sass/views/_maintenance.scss +++ /dev/null @@ -1,104 +0,0 @@ -.maintenance-header { - text-align: center; - margin-top: 50px; - - h2 { - margin-bottom: 10px; - } -} - -.maintenance-content { - padding: 3rem 0; - - .maintenance-list { - max-width: 1280px; - margin: 0 auto; - - .view-list-container { - padding: 10px 15px; - background-color: #fff; - border-bottom: 1px solid #ddd; - - &:hover { - background-color: #fafafa; - } - - .view-name { - display: inline-block; - width: 20%; - float: left; - } - - .view-desc { - display: inline-block; - width: 80%; - font-size: 15px; - } - } - } - - .maintenance-form { - width: 60%; - margin: auto; - - .result-list { - height: calc(100vh - 200px); - overflow: auto; - } - - .result { - box-shadow: inset 0 1px 2px rgba(0, 0, 0, 0.2); - margin-top: 15px; - padding: 15px 30px; - background: #f9f9f9; - } - - li { - font-size: 13px; - line-height: 9px; - } - - .actions { - text-align: right; - } - - .field-radio div { - display: inline-block; - margin-right: 10px; - } - - div.error { - color: #f00; - margin-top: 10px; - font-size: 13px; - } - - div.head-output { - font-size: 13px; - margin-bottom: 10px; - } - - div.main-output { - color: #0a0; - font-size: 15px; - } - } - - .announcement-container { - width: 100%; - text-align: center; - - .announcement-item { - display: inline-block; - max-width: 300px; - min-width: 300px; - margin: 15px; - - .announcement-content { - background-color: $body-bg; - text-align: center; - padding: 22px 33px; - } - } - } -} diff --git a/cms/templates/maintenance/_announcement_delete.html b/cms/templates/maintenance/_announcement_delete.html deleted file mode 100644 index 0397ef5a0bf8..000000000000 --- a/cms/templates/maintenance/_announcement_delete.html +++ /dev/null @@ -1,40 +0,0 @@ -<%page expression_filter="h"/> -<%inherit file="base.html" /> -<%namespace name='static' file='../static_content.html'/> -<%! -from django.utils.translation import gettext as _ -from openedx.core.djangolib.markup import HTML, Text -%> -<%block name="title">${_('Delete Announcement')} -<%block name="viewtitle"> -

- ${_('Delete Announcement')} -

- - -<%block name="viewcontent"> -
-
-
- -
-
- -
-
- ## xss-lint: disable=mako-invalid-html-filter - ${object.content | n} -
-
-
- -
-
-
-
- diff --git a/cms/templates/maintenance/_announcement_edit.html b/cms/templates/maintenance/_announcement_edit.html deleted file mode 100644 index a9bee1c6fce2..000000000000 --- a/cms/templates/maintenance/_announcement_edit.html +++ /dev/null @@ -1,50 +0,0 @@ -<%page expression_filter="h"/> -<%inherit file="base.html" /> -<%namespace name='static' file='../static_content.html'/> -<%! -from django.utils.translation import gettext as _ -from openedx.core.djangolib.markup import HTML, Text -%> -<%block name="title">${_('Edit Announcement')} -<%block name="viewtitle"> -

- ${_('Edit Announcement')} -

- - -<%block name="viewcontent"> -
-
-
-
- -
- ## xss-lint: disable=mako-invalid-html-filter - ${form.as_p() | n} -
-
- -
-
-
-
-
- - -<%block name="header_extras"> - - - diff --git a/cms/templates/maintenance/_announcement_index.html b/cms/templates/maintenance/_announcement_index.html deleted file mode 100644 index 68713c9986cc..000000000000 --- a/cms/templates/maintenance/_announcement_index.html +++ /dev/null @@ -1,59 +0,0 @@ -<%page expression_filter="h"/> -<%namespace name='static' file='../static_content.html'/> -<%! -from django.urls import reverse -from django.utils.translation import gettext as _ - -from openedx.core.djangolib.markup import HTML, Text - -%> -
-
-
- % for announcement in announcement_list: -
-
- ## xss-lint: disable=mako-invalid-html-filter - ${announcement.content | n} -
- -
- - % if announcement.active: - Active
-
- % endfor -
-
- - - - % if is_paginated: - % if page_obj.has_previous(): - - - - % endif - - % if page_obj.has_next(): - - - - % endif - % endif -
-
-
diff --git a/cms/templates/maintenance/base.html b/cms/templates/maintenance/base.html deleted file mode 100644 index 6979797a629c..000000000000 --- a/cms/templates/maintenance/base.html +++ /dev/null @@ -1,21 +0,0 @@ -<%page expression_filter="h"/> -<%inherit file="../base.html" /> -<%def name='online_help_token()'><% return 'maintenance' %> -<%namespace name='static' file='../static_content.html'/> -<%! -from django.urls import reverse -from django.utils.translation import gettext as _ -%> -<%block name="content"> -
-
-

- - ${_('Maintenance Dashboard')} - -

- <%block name="viewtitle"> - -
-<%block name="viewcontent"> - diff --git a/cms/templates/maintenance/container.html b/cms/templates/maintenance/container.html deleted file mode 100644 index 319a57bfe995..000000000000 --- a/cms/templates/maintenance/container.html +++ /dev/null @@ -1,25 +0,0 @@ -<%page expression_filter="h"/> -<%inherit file="base.html" /> -<%namespace name='static' file='../static_content.html'/> -<%! -from django.urls import reverse -from openedx.core.djangolib.js_utils import js_escaped_string -%> -<%block name="title">${view['name']} -<%block name="viewtitle"> -

- ${view['name']} -

- - -<%block name="viewcontent"> -
- <%include file="_${view['slug']}.html"/> -
- - -<%block name="requirejs"> - require(["js/maintenance/${view['slug'] | n, js_escaped_string}"], function(MaintenanceFactory) { - MaintenanceFactory("${reverse(view['url']) | n, js_escaped_string}"); - }); - diff --git a/cms/templates/maintenance/index.html b/cms/templates/maintenance/index.html deleted file mode 100644 index 293cb90b4a9c..000000000000 --- a/cms/templates/maintenance/index.html +++ /dev/null @@ -1,20 +0,0 @@ -<%page expression_filter="h"/> -<%inherit file="base.html" /> -<%namespace name='static' file='../static_content.html'/> -<%! -from django.utils.translation import gettext as _ -from django.urls import reverse -%> -<%block name="title">${_('Maintenance Dashboard')} -<%block name="viewcontent"> -
-
    - % for view in views.values(): -
  • - ${view['name']} - ${view['description']} -
    • - % endfor -
-
- diff --git a/cms/templates/widgets/user_dropdown.html b/cms/templates/widgets/user_dropdown.html index 0ec00257ffe1..3fc0934b0db7 100644 --- a/cms/templates/widgets/user_dropdown.html +++ b/cms/templates/widgets/user_dropdown.html @@ -21,11 +21,6 @@

- % if GlobalStaff().has_user(user): -
  • - ${_("Maintenance")} -
    • - % endif diff --git a/cms/urls.py b/cms/urls.py index c60b56c3bd49..048339bc9fe9 100644 --- a/cms/urls.py +++ b/cms/urls.py @@ -284,8 +284,6 @@ certificates_list_handler, name='certificates_list_handler') ] -# Maintenance Dashboard -urlpatterns.append(path('maintenance/', include('cms.djangoapps.maintenance.urls', namespace='maintenance'))) if settings.DEBUG: try: diff --git a/lms/static/sass/_build-lms-v1.scss b/lms/static/sass/_build-lms-v1.scss index 4d64e6768515..90c5077c1f38 100644 --- a/lms/static/sass/_build-lms-v1.scss +++ b/lms/static/sass/_build-lms-v1.scss @@ -67,7 +67,6 @@ // features @import 'features/bookmarks-v1'; -@import "features/announcements"; @import 'features/_unsupported-browser-alert'; @import 'features/content-type-gating'; @import 'features/course-duration-limits'; diff --git a/lms/static/sass/features/_announcements.scss b/lms/static/sass/features/_announcements.scss deleted file mode 100644 index 0c3c01fe6077..000000000000 --- a/lms/static/sass/features/_announcements.scss +++ /dev/null @@ -1,28 +0,0 @@ -// lms - features - announcements -// ==================== -.announcements-list { - display: inline-block; - width: 100%; - - .announcement { - background-color: $course-profile-bg; - align-content: center; - text-align: center; - padding: 22px 33px; - margin-bottom: 15px; - } - - .announcement-button { - display: inline-block; - padding: 3px 10px; - font-size: 0.75rem; - } - - .prev { - float: left; - } - - .next { - float: right; - } -} diff --git a/lms/templates/dashboard.html b/lms/templates/dashboard.html index d44cdefc6389..6149418035ca 100644 --- a/lms/templates/dashboard.html +++ b/lms/templates/dashboard.html @@ -294,16 +294,6 @@

    ${course_dir}

    % endif - - <%block name="skip_links"> - % if settings.FEATURES.get('ENABLE_ANNOUNCEMENTS'): - ${_("Skip to list of announcements")} - % endif - - % if settings.FEATURES.get('ENABLE_ANNOUNCEMENTS'): - <%include file='dashboard/_dashboard_announcements.html' /> - % endif -
    diff --git a/openedx/features/announcements/__init__.py b/openedx/features/announcements/__init__.py deleted file mode 100644 index e69de29bb2d1..000000000000 diff --git a/openedx/features/announcements/apps.py b/openedx/features/announcements/apps.py deleted file mode 100644 index 4bf964cae51b..000000000000 --- a/openedx/features/announcements/apps.py +++ /dev/null @@ -1,32 +0,0 @@ -""" -Announcements Application Configuration -""" - - -from django.apps import AppConfig -from edx_django_utils.plugins import PluginURLs, PluginSettings - -from openedx.core.djangoapps.plugins.constants import ProjectType, SettingsType - - -class AnnouncementsConfig(AppConfig): - """ - Application Configuration for Announcements - """ - name = 'openedx.features.announcements' - - plugin_app = { - PluginURLs.CONFIG: { - ProjectType.LMS: { - PluginURLs.NAMESPACE: 'announcements', - PluginURLs.REGEX: '^announcements/', - PluginURLs.RELATIVE_PATH: 'urls', - } - }, - PluginSettings.CONFIG: { - ProjectType.LMS: { - SettingsType.COMMON: {PluginSettings.RELATIVE_PATH: 'settings.common'}, - SettingsType.TEST: {PluginSettings.RELATIVE_PATH: 'settings.test'}, - } - } - } diff --git a/openedx/features/announcements/forms.py b/openedx/features/announcements/forms.py deleted file mode 100644 index 879101ca37d0..000000000000 --- a/openedx/features/announcements/forms.py +++ /dev/null @@ -1,20 +0,0 @@ -""" -Forms for the Announcement Editor -""" - - -from django import forms - -from .models import Announcement - - -class AnnouncementForm(forms.ModelForm): - """ - Form for editing Announcements - """ - content = forms.CharField(widget=forms.Textarea, label='', required=False) - active = forms.BooleanField(initial=True, required=False) - - class Meta: - model = Announcement - fields = ['content', 'active'] diff --git a/openedx/features/announcements/migrations/0001_initial.py b/openedx/features/announcements/migrations/0001_initial.py deleted file mode 100644 index c959b634905e..000000000000 --- a/openedx/features/announcements/migrations/0001_initial.py +++ /dev/null @@ -1,18 +0,0 @@ -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ] - - operations = [ - migrations.CreateModel( - name='Announcement', - fields=[ - ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), - ('content', models.CharField(default='lorem ipsum', max_length=1000)), - ('active', models.BooleanField(default=True)), - ], - ), - ] diff --git a/openedx/features/announcements/migrations/__init__.py b/openedx/features/announcements/migrations/__init__.py deleted file mode 100644 index e69de29bb2d1..000000000000 diff --git a/openedx/features/announcements/models.py b/openedx/features/announcements/models.py deleted file mode 100644 index f58f61165db6..000000000000 --- a/openedx/features/announcements/models.py +++ /dev/null @@ -1,22 +0,0 @@ -""" -Models for Announcements -""" - - -from django.db import models - - -class Announcement(models.Model): - """ - Site-wide announcements to be displayed on the dashboard - - .. no_pii: - """ - class Meta: - app_label = 'announcements' - - content = models.CharField(max_length=1000, null=False, default="lorem ipsum") - active = models.BooleanField(default=True) - - def __str__(self): - return self.content diff --git a/openedx/features/announcements/settings/__init__.py b/openedx/features/announcements/settings/__init__.py deleted file mode 100644 index e69de29bb2d1..000000000000 diff --git a/openedx/features/announcements/settings/common.py b/openedx/features/announcements/settings/common.py deleted file mode 100644 index 4de3740d2d27..000000000000 --- a/openedx/features/announcements/settings/common.py +++ /dev/null @@ -1,21 +0,0 @@ -"""Common settings for Announcements""" - - -def plugin_settings(settings): - """ - Common settings for Announcements - .. toggle_name: FEATURES['ENABLE_ANNOUNCEMENTS'] - .. toggle_implementation: SettingDictToggle - .. toggle_default: False - .. toggle_description: This feature can be enabled to show system wide announcements - on the sidebar of the learner dashboard. Announcements can be created by Global Staff - users on maintenance dashboard of studio. Maintenance dashboard can accessed at - https://{studio.domain}/maintenance - .. toggle_warning: TinyMCE is needed to show an editor in the studio. - .. toggle_use_cases: open_edx - .. toggle_creation_date: 2017-11-08 - .. toggle_tickets: https://github.com/openedx/edx-platform/pull/16496 - """ - settings.ENABLE_ANNOUNCEMENTS = False - # Configure number of announcements to show per page - settings.ANNOUNCEMENTS_PER_PAGE = 5 diff --git a/openedx/features/announcements/settings/test.py b/openedx/features/announcements/settings/test.py deleted file mode 100644 index 8c8406d23f4b..000000000000 --- a/openedx/features/announcements/settings/test.py +++ /dev/null @@ -1,8 +0,0 @@ -"""Test settings for Announcements""" - - -def plugin_settings(settings): - """ - Test settings for Announcements - """ - settings.ENABLE_ANNOUNCEMENTS = True diff --git a/openedx/features/announcements/static/announcements/jsx/Announcements.jsx b/openedx/features/announcements/static/announcements/jsx/Announcements.jsx deleted file mode 100644 index 9d370883352c..000000000000 --- a/openedx/features/announcements/static/announcements/jsx/Announcements.jsx +++ /dev/null @@ -1,141 +0,0 @@ -// eslint-disable-next-line max-classes-per-file -import React from 'react'; -import ReactDOM from 'react-dom'; -import PropTypes from 'prop-types'; -import {Button} from '@edx/paragon'; -import $ from 'jquery'; - -class AnnouncementSkipLink extends React.Component { - constructor(props) { - super(props); - this.state = { - count: 0 - }; - $.get('/announcements/page/1') - .then(data => { - this.setState({ - count: data.count - }); - }); - } - - render() { - return (
    {'Skip to list of ' + this.state.count + ' announcements'}
    ); - } -} - -// eslint-disable-next-line react/prefer-stateless-function -class Announcement extends React.Component { - render() { - return ( -
    - ); - } -} - -Announcement.propTypes = { - content: PropTypes.string.isRequired, -}; - -class AnnouncementList extends React.Component { - constructor(props) { - super(props); - this.state = { - page: 1, - announcements: [], - // eslint-disable-next-line react/no-unused-state - num_pages: 0, - has_prev: false, - has_next: false, - start_index: 0, - end_index: 0, - }; - } - - retrievePage(page) { - $.get('/announcements/page/' + page) - .then(data => { - this.setState({ - announcements: data.announcements, - has_next: data.next, - has_prev: data.prev, - // eslint-disable-next-line react/no-unused-state - num_pages: data.num_pages, - count: data.count, - start_index: data.start_index, - end_index: data.end_index, - page: page - }); - }); - } - - renderPrevPage() { - this.retrievePage(this.state.page - 1); - } - - renderNextPage() { - this.retrievePage(this.state.page + 1); - } - - // eslint-disable-next-line react/no-deprecated, react/sort-comp - componentWillMount() { - this.retrievePage(this.state.page); - } - - render() { - var children = this.state.announcements.map( - // eslint-disable-next-line react/no-array-index-key - (announcement, index) => - ); - if (this.state.has_prev) { - var prev_button = ( -
    -
    - ); - } - if (this.state.has_next) { - var next_button = ( -
    -
    - ); - } - return ( -
    - {children} - {prev_button} - {next_button} -
    - ); - } -} - -export default class AnnouncementsView { - constructor() { - ReactDOM.render( - , - document.getElementById('announcements'), - ); - ReactDOM.render( - , - document.getElementById('announcements-skip'), - ); - } -} - -export {AnnouncementsView, AnnouncementList, AnnouncementSkipLink}; diff --git a/openedx/features/announcements/static/announcements/jsx/Announcements.test.jsx b/openedx/features/announcements/static/announcements/jsx/Announcements.test.jsx deleted file mode 100644 index 3ec55f392889..000000000000 --- a/openedx/features/announcements/static/announcements/jsx/Announcements.test.jsx +++ /dev/null @@ -1,25 +0,0 @@ -import React from 'react'; -import renderer from 'react-test-renderer'; -import testAnnouncements from './test-announcements.json'; - -import {AnnouncementSkipLink, AnnouncementList} from './Announcements'; - -describe('Announcements component', () => { - test('render skip link', () => { - const component = renderer.create( - , - ); - component.root.instance.setState({count: 10}); - const tree = component.toJSON(); - expect(tree).toMatchSnapshot(); - }); - - test('render test announcements', () => { - const component = renderer.create( - , - ); - component.root.instance.setState(testAnnouncements); - const tree = component.toJSON(); - expect(tree).toMatchSnapshot(); - }); -}); diff --git a/openedx/features/announcements/static/announcements/jsx/__snapshots__/Announcements.test.jsx.snap b/openedx/features/announcements/static/announcements/jsx/__snapshots__/Announcements.test.jsx.snap deleted file mode 100644 index bbf9bfaaaa69..000000000000 --- a/openedx/features/announcements/static/announcements/jsx/__snapshots__/Announcements.test.jsx.snap +++ /dev/null @@ -1,78 +0,0 @@ -// Jest Snapshot v1, https://goo.gl/fbAQLP - -exports[`Announcements component render skip link 1`] = ` -
    - Skip to list of 10 announcements -
    -`; - -exports[`Announcements component render test announcements 1`] = ` -
    -
    -
    Announcement 2", - } - } - /> -
    -
    -
    -
    -
    - - - 1 - 5) of 6 - -
    -
    -`; diff --git a/openedx/features/announcements/static/announcements/jsx/test-announcements.json b/openedx/features/announcements/static/announcements/jsx/test-announcements.json deleted file mode 100644 index d23d39303020..000000000000 --- a/openedx/features/announcements/static/announcements/jsx/test-announcements.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "announcements": [ - {"content": "Test Announcement 1"}, - {"content": "Bold Announcement 2"}, - {"content": "Test Announcement 3"}, - {"content": "Test Announcement 4"}, - {"content": "Test Announcement 5"}, - {"content": "Test Announcement 6"} - ], - "has_next": true, - "has_prev": false, - "num_pages": 2, - "count": 6, - "start_index": 1, - "end_index": 5, - "page": 1 -} diff --git a/openedx/features/announcements/tests/__init__.py b/openedx/features/announcements/tests/__init__.py deleted file mode 100644 index e69de29bb2d1..000000000000 diff --git a/openedx/features/announcements/tests/test_announcements.py b/openedx/features/announcements/tests/test_announcements.py deleted file mode 100644 index 10c608b4a6cd..000000000000 --- a/openedx/features/announcements/tests/test_announcements.py +++ /dev/null @@ -1,95 +0,0 @@ -""" -Unit tests for the announcements feature. -""" - -import json -from unittest.mock import patch - -from django.conf import settings -from django.test import TestCase -from django.test.client import Client -from django.urls import reverse - -from common.djangoapps.student.tests.factories import AdminFactory -from openedx.core.djangolib.testing.utils import skip_unless_lms -from openedx.features.announcements.models import Announcement - -TEST_ANNOUNCEMENTS = [ - ("Active Announcement", True), - ("Inactive Announcement", False), - ("Another Test Announcement", True), - ("Formatted Announcement", True), - ("Other Formatted Announcement", True), -] - - -@skip_unless_lms -class TestGlobalAnnouncements(TestCase): - """ - Test Announcements in LMS - """ - - @classmethod - def setUpTestData(cls): - super().setUpTestData() - Announcement.objects.bulk_create([ - Announcement(content=content, active=active) - for content, active in TEST_ANNOUNCEMENTS - ]) - - def setUp(self): - super().setUp() - self.client = Client() - self.admin = AdminFactory.create( - email='staff@edx.org', - username='admin', - password='pass' - ) - self.client.login(username=self.admin.username, password='pass') - - @patch.dict(settings.FEATURES, {'ENABLE_ANNOUNCEMENTS': False}) - def test_feature_flag_disabled(self): - """Ensures that the default settings effectively disables the feature""" - response = self.client.get('/dashboard') - self.assertNotContains(response, 'AnnouncementsView') - self.assertNotContains(response, '
    Formatted Announcement") diff --git a/openedx/features/announcements/urls.py b/openedx/features/announcements/urls.py deleted file mode 100644 index 0f0ad3a33960..000000000000 --- a/openedx/features/announcements/urls.py +++ /dev/null @@ -1,13 +0,0 @@ -""" -Defines URLs for announcements in the LMS. -""" -from django.contrib.auth.decorators import login_required -from django.urls import path - -from .views import AnnouncementsJSONView - -urlpatterns = [ - path('page/', login_required(AnnouncementsJSONView.as_view()), - name='page', - ), -] diff --git a/openedx/features/announcements/views.py b/openedx/features/announcements/views.py deleted file mode 100644 index b6657c29cc12..000000000000 --- a/openedx/features/announcements/views.py +++ /dev/null @@ -1,37 +0,0 @@ -""" -Views to show announcements. -""" - - -from django.conf import settings -from django.http import JsonResponse -from django.views.generic.list import ListView - -from .models import Announcement - - -class AnnouncementsJSONView(ListView): - """ - View returning a page of announcements for the dashboard - """ - model = Announcement - object_list = Announcement.objects.filter(active=True) - paginate_by = settings.FEATURES.get('ANNOUNCEMENTS_PER_PAGE', 5) - - def get(self, request, *args, **kwargs): - """ - Return active announcements as json - """ - context = self.get_context_data() - - announcements = [{"content": announcement.content} for announcement in context['object_list']] - result = { - "announcements": announcements, - "next": context['page_obj'].has_next(), - "prev": context['page_obj'].has_previous(), - "start_index": context['page_obj'].start_index(), - "end_index": context['page_obj'].end_index(), - "count": context['paginator'].count, - "num_pages": context['paginator'].num_pages, - } - return JsonResponse(result) diff --git a/setup.py b/setup.py index 5b9f020ac3c5..eeb7b79f534d 100644 --- a/setup.py +++ b/setup.py @@ -139,7 +139,6 @@ ], "lms.djangoapp": [ "ace_common = openedx.core.djangoapps.ace_common.apps:AceCommonConfig", - "announcements = openedx.features.announcements.apps:AnnouncementsConfig", "content_libraries = openedx.core.djangoapps.content_libraries.apps:ContentLibrariesConfig", "course_apps = openedx.core.djangoapps.course_apps.apps:CourseAppsConfig", "course_live = openedx.core.djangoapps.course_live.apps:CourseLiveConfig", @@ -158,7 +157,6 @@ "program_enrollments = lms.djangoapps.program_enrollments.apps:ProgramEnrollmentsConfig", ], "cms.djangoapp": [ - "announcements = openedx.features.announcements.apps:AnnouncementsConfig", "ace_common = openedx.core.djangoapps.ace_common.apps:AceCommonConfig", "bookmarks = openedx.core.djangoapps.bookmarks.apps:BookmarksConfig", "course_live = openedx.core.djangoapps.course_live.apps:CourseLiveConfig", diff --git a/webpack.common.config.js b/webpack.common.config.js index f6fd4fa357f2..ac283c3d40d8 100644 --- a/webpack.common.config.js +++ b/webpack.common.config.js @@ -134,7 +134,6 @@ module.exports = Merge.merge({ // Features Currency: './openedx/features/course_experience/static/course_experience/js/currency.js', - AnnouncementsView: './openedx/features/announcements/static/announcements/jsx/Announcements.jsx', CookiePolicyBanner: './common/static/js/src/CookiePolicyBanner.jsx', // Common From 515a3017f209deaf8cd3f644cf86e2b56c783543 Mon Sep 17 00:00:00 2001 From: Glib Glugovskiy Date: Fri, 10 Oct 2025 20:11:15 +0300 Subject: [PATCH 010/351] docs: ADR introducing mobile offline content support (#35011) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Іван Нєдєльніцев Co-authored-by: Kyrylo Kholodenko Co-authored-by: Glib Glugovskiy --- .../001-mobile-offline-content-support.rst | 156 ++++++++++++++++++ .../002-mobile-offline-content-support.rst | 55 ++++++ .../mobile_offline_content_generation.svg | 4 + 3 files changed, 215 insertions(+) create mode 100644 openedx/features/offline_content/docs/001-mobile-offline-content-support.rst create mode 100644 openedx/features/offline_content/docs/002-mobile-offline-content-support.rst create mode 100644 openedx/features/offline_content/docs/_images/mobile_offline_content_generation.svg diff --git a/openedx/features/offline_content/docs/001-mobile-offline-content-support.rst b/openedx/features/offline_content/docs/001-mobile-offline-content-support.rst new file mode 100644 index 000000000000..29ecf7d01efe --- /dev/null +++ b/openedx/features/offline_content/docs/001-mobile-offline-content-support.rst @@ -0,0 +1,156 @@ +1. Offline content generation for mobile OeX app +============================================= + +Status +------ + +Accepted + +Context +------- + +The primary goal is to enable offline access to course content in the Open edX mobile application. +This will allow users to download course materials when they have internet access and access them +later without an internet connection, also it should support synchronization of the submitted results +with backend service as connection become available again. This feature is crucial for learners +in areas with unreliable internet connectivity or those who prefer to study on the go without using mobile data. +It is possible to provide different kind of content using the Open edX platform, such as read-only materials, +videos, and assessments. Therefore to provide the whole course experience in offline mode it's required to +make all these types of content available offline. Of course it won't be feasible to recreate grading +algorithms in mobile, so it's possible to save submission on the mobile app and execute synchronization +of the user progres as not limited connectivity is back. + +From the product perspective the following Figma designs and product requirements should be considered: + +* `Download and Delete (Figma)`_ +* `Downloads (Figma)`_ + +.. _Download and Delete (Figma): https://www.figma.com/design/iZ56YMjbRMShCCDxqrqRrR/Mobile-App-v2.4-%5BOpen-edX%5D?node-id=18472-187387&t=tMgymS6WIZZJbJHn-0 +.. _Downloads (Figma): https://www.figma.com/design/iZ56YMjbRMShCCDxqrqRrR/Mobile-App-v2.4-%5BOpen-edX%5D + +Decision +-------- + +The implementation of the offline content support require addition of the following features to the edx-platform: + +* It's necessary to generate an archive with all necessary HTML and assets for a student view of an xBlock, so it's possible to display an xBlock using mobile WebView. +* Implement a new standard XBlock view called `offline_view` which would generate user-agnostic fragments suitable for offline use. This view will avoid any dependence on student-specific state, focusing solely on content and settings. +* XBlock classes can opt into supporting `offline_view`. They can implement this view fully or partially. For example, a block that relies on user-specific randomization or interactive elements that require online connectivity would not be rendered offline. +* The generated offline content should be provided to mobile device through mobile API. +* To support CAPA problems and other kinds of assessments in offline mode it's necessary to create an additional + JavaScript layer that will allow communication with Mobile applications by sending JSON messages + using Android and IOS Bridge. + + +Offline content generation +~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Generating zip archive with xBlock data for HTML and CAPA problems +When content is published in CMS and offline generation is enabled for the course or entire platform using waffle flags, the content generation task should be started for supported blocks. +Every time block content republished ZIP archive with offline content should be regenerated. +Supported XBlock class should implement `offline_view` method that will be used to generate the content. +HTML should be processed, all related assets files, images and scripts should be included in the generated ZIP archive with offline content +The Generation process should work with local media storage as well as s3. +If error retrieving block happened, the generation task will be scheduled for retry 2 more times, with progressive delay. + + .. image:: _images/mobile_offline_content_generation.svg + :alt: Mobile Offline Content Generation Process Diagram + + +Offline content deletion +~~~~~~~~~~~~~~~~~~~~~~~~ + +When the course is published and some blocks are removed from the course, related ZIP archive should be deleted. +When some blocks are removed from the course without publishing the course, the related ZIP archive shouldn't be deleted. + + +Mobile API extension +~~~~~~~~~~~~~~~~~~~~ + +Extend the Course Home mobile API endpoint, and add a new version of the API (url /api/mobile/v4/course_info/blocks/) +to return information about offline content available for download for supported blocks + +.. code-block:: json + { + ... + "offline_download": { + "file_url": "{file_url}" or null, + "last_modified": "{DT}" or null, + "file_size": "" + } + } + + +JavaScript Bridge for interaction with mobile applications +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Implement JS Bridge JS script to intercept and send results to mobile device for supported CAPA problems. + +The JS bridge will intercept AJAX requests in the mobile application and store the responses locally. If the user submits the response offline he will be shown the message "Your response is accepted" and the Submit button will be disabled as the submission will be sent twice. +When the internet connection is back the mobile client will submit the cached responses one by one through the regular xBlock handler endpoints. +Data from submission should be submitted through bridge on iOS and Android devices. +This script should expose markCompleted JS function so mobile can change state of the offline problem after the data was saved into internal database or on initialization of the problem. + +* **Implement of a mechanism for generating and storing on a server or external storage**: The course content should be pre-generated and saved to the storage for later download. + * **Render block fragment**: Implement a new standard XBlock view called `offline_view` which would generate user-agnostic fragments suitable for offline use. This view will avoid any dependence on student-specific state, focusing solely on content and settings. + * **Replace static and media**: Save static and media assets files used in block to temporary directory and replace their static paths with local paths. + * **Archive and store content**: Archive the generated content and store it on the server or external storage. +* **Mechanism for updating the generated data**: When updating course blocks (namely when publishing) the content that has been changed should be re-generated. + * **Track course publishing events on CMS side**: Add a new signal `course_cache_updated` to be called after the course structure cache update in `update_course_in_cache_v2`. Add a signal that listens to `course_cache_updated` and starts block generation. + * **Update archive**: Check generated archive creation date and update it if less than course publishing date. +* **Implement a Mobile Local Storage Mechanism**: Use the device's local storage to save course content for offline access. + * **Extend blocks API**: Add links to download blocks content and where it is possible. +* **Sync Mechanism**: Periodically synchronize local data with the server when the device is online. + * **Sync on app side**: On course outline screen, check if the course content is up to date and update it if necessary. + * **Sync user responses**: When the device is offline, save user responses locally and send them to the server when the device is online. +* **Selective Download**: Allow users to choose specific content to download for offline use. +* **Full Course Download**: Provide an option to download entire courses for offline access. + +Supported xBlocks in offline mode +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +It was decided to include a fraction of Open edX xBlocks to be supported. +The following list of blocks is currently planned to be added to the support: + +* **Common problems**: + * **Checkboxes** - full support + * **Dropdown** - full support + * **Multiple Choice** - full support + * **Numerical Input** - full support + * **Text Input** - full support + * **Checkboxes with Hints and Feedback** - partial support without Hints and Feedback + * **Dropdown with Hints and Feedback** - partial support without Hints and Feedback + * **Multiple Choice with Hints and Feedback** - partial support without Hints and Feedback + * **Numerical Input with Hints and Feedback** - partially supported without Hints and Feedback + * **Text Input with Hints and Feedback** - partially supported without Hints and Feedback + * **Blank Advanced Problems** - partially supported, without loncapa/python problems or multi-part problems +* **Text**: + * **Text** - full support + * **IFrame Tool** - full support + * **Raw HTML** - full support + * **Zooming Image Tool** - full support +* **Video** - already supported + + +Consequences +------------ + +* Enhanced learner experience with flexible access to course materials. +* Increased accessibility for learners in regions with poor internet connectivity. +* Improved engagement and completion rates due to uninterrupted access to content. +* Simplified Maintenance by using a unified rendering view (`offline_view`), the complexity of maintaining separate renderers for online and offline content is significantly reduced. +* The proposed approach not only caters to the current needs of mobile users but also sets a foundation for expanding offline access to other platforms and uses. +* Potential increase in app size due to locally stored content. +* Increased complexity in managing content synchronization and updates. +* Need for continuous monitoring and updates to handle new content types and formats. + +Rejected Solutions +------------------ + +* **Store common .js and .css files of blocks in a separate folder:** + * This solution was rejected because it is unclear how to track potential changes to these files and re-generate the content of the blocks. + +* **Generate content on the fly when the user requests it:** + * This solution was rejected because it would require a significant amount of processing power and time to generate content for each block when requested. + +* **Separate Offline Renderer**: + * The initial proposal of creating a separate renderer for offline content was rejected due to the increased complexity and potential for inconsistent behavior between online and offline content. diff --git a/openedx/features/offline_content/docs/002-mobile-offline-content-support.rst b/openedx/features/offline_content/docs/002-mobile-offline-content-support.rst new file mode 100644 index 000000000000..7d1e91e7e882 --- /dev/null +++ b/openedx/features/offline_content/docs/002-mobile-offline-content-support.rst @@ -0,0 +1,55 @@ +2. Offline Mode enhancements +========================= + +Status +------ + +Proposed + +Context +------- + +`offline_view` generalized and can be used for Non-mobile offline mode, Anonymous access or Regular student access. +Static files like JavaScript and CSS will be de-duplicated based on their content hash. + +Decisions +-------- + +1. Efficient resource management +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + - Shared resources like JS and CSS files will be de-duplicated based on their content hash, to prevent duplication for every block. + - All shared content should be stored in the separate ZIP archive. + - This archive will be regenerated 1 time and contains all JS and CSS files related to default Xblocks. + - Xblock specific resources will still be stored in the block ZIP archive. + - This will ensure that the same resource is not duplicated across different blocks, reducing storage and bandwidth usage. + + +2. Anonymous access +~~~~~~~~~~~~~~~~~~~ + + - Re-implement `public_view` on top of `offline_view`. If it is possible to get pre-rendered block without knowing user state, then it is possible to serve that pre-renderable view as the public experience for logged-out users. + - This will allow broader access to educational content without the need for user authentication, potentially increasing user engagement and content reach. + + +3. Non-mobile offline mode +~~~~~~~~~~~~~~~~~~~~~~~~~~ + + - The `offline_view` will be generalized to support non-mobile offline mode. + - This mode will enable users on desktop and other non-mobile platforms to download and access course content without an active internet connection, providing greater flexibility in how content is accessed. + + +4. Regular student access +~~~~~~~~~~~~~~~~~~~~~~~~~ + + - `student_view` will be implemented on top of `offline_view` wherever it is supported. + - For XBlocks compatible with this architecture, offline-ready content will be served by default, and dynamic online features will be engaged only when a user has a reliable connection. + - This setting is intended to improve the learning process by providing constant access to content when the Internet connection is unstable. + + +Consequences +------------ + +* **Resource Efficiency**: The avoidance of duplicating static resources for each block enhances the efficient use of storage and bandwidth. +* **Enhanced Flexibility**: The system can skip rendering blocks that require student-specific interactions, ensuring reliability and reducing the potential for behavior discrepancies between online and offline modes. +* **Broader Accessibility**: The ability to serve pre-rendered views to anonymous users increases the reach of educational content, making it more accessible to a wider audience. diff --git a/openedx/features/offline_content/docs/_images/mobile_offline_content_generation.svg b/openedx/features/offline_content/docs/_images/mobile_offline_content_generation.svg new file mode 100644 index 000000000000..ca07617294a9 --- /dev/null +++ b/openedx/features/offline_content/docs/_images/mobile_offline_content_generation.svg @@ -0,0 +1,4 @@ + + +CMSExternal media storageChange/create coursecontentPublish changesGenerate course xblock filesOffline content generation handlerRun celery task togenerate xblocks forcourseChecking whether thecontent of aspecific xblockneeds to begenerated (re-generated)Creating a temporarydir to store xblockcontentCall offline_viewfor xblockRendering xblockHTML file viaCMS rendererCopy allrelated staticfiles (js andcss) for therendered xblockArchive xblocktemporary dir,and itsdeletionCourses offline contentOther OeX mediaCourse 1 folderProblem xblock archiveHTML xblock archive....Course 2 folderProblem xblock archiveHTML xblock archive........YesSave xblockarchive tothe storageСourse cache updates \ No newline at end of file From 467bb32b66ac469df4437322f5c2445385bfb917 Mon Sep 17 00:00:00 2001 From: iloveagent57 <2307986+iloveagent57@users.noreply.github.com> Date: Fri, 10 Oct 2025 13:22:26 +0000 Subject: [PATCH 011/351] feat: Upgrade Python dependency edx-enterprise feat: add an endpoint to create a customer admin user Commit generated by workflow `openedx/edx-platform/.github/workflows/upgrade-one-python-dependency.yml@refs/heads/master` --- requirements/constraints.txt | 2 +- requirements/edx/base.txt | 2 +- requirements/edx/development.txt | 2 +- requirements/edx/doc.txt | 2 +- requirements/edx/testing.txt | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/requirements/constraints.txt b/requirements/constraints.txt index 8ae221c9dadd..3525cdab24db 100644 --- a/requirements/constraints.txt +++ b/requirements/constraints.txt @@ -42,7 +42,7 @@ django-stubs<6 # The team that owns this package will manually bump this package rather than having it pulled in automatically. # This is to allow them to better control its deployment and to do it in a process that works better # for them. -edx-enterprise==6.4.4 +edx-enterprise==6.5.0 # Date: 2023-07-26 # Our legacy Sass code is incompatible with anything except this ancient libsass version. diff --git a/requirements/edx/base.txt b/requirements/edx/base.txt index be367e3d8972..6cb36c1986fb 100644 --- a/requirements/edx/base.txt +++ b/requirements/edx/base.txt @@ -473,7 +473,7 @@ edx-drf-extensions==10.6.0 # edxval # enterprise-integrated-channels # openedx-learning -edx-enterprise==6.4.4 +edx-enterprise==6.5.0 # via # -c requirements/constraints.txt # -r requirements/edx/kernel.in diff --git a/requirements/edx/development.txt b/requirements/edx/development.txt index 136ae43a6f6c..a1e35b928940 100644 --- a/requirements/edx/development.txt +++ b/requirements/edx/development.txt @@ -747,7 +747,7 @@ edx-drf-extensions==10.6.0 # edxval # enterprise-integrated-channels # openedx-learning -edx-enterprise==6.4.4 +edx-enterprise==6.5.0 # via # -c requirements/constraints.txt # -r requirements/edx/doc.txt diff --git a/requirements/edx/doc.txt b/requirements/edx/doc.txt index 6310ee6cece0..cb280ad45e90 100644 --- a/requirements/edx/doc.txt +++ b/requirements/edx/doc.txt @@ -557,7 +557,7 @@ edx-drf-extensions==10.6.0 # edxval # enterprise-integrated-channels # openedx-learning -edx-enterprise==6.4.4 +edx-enterprise==6.5.0 # via # -c requirements/constraints.txt # -r requirements/edx/base.txt diff --git a/requirements/edx/testing.txt b/requirements/edx/testing.txt index 895c92da1c9f..9a0d5e839de3 100644 --- a/requirements/edx/testing.txt +++ b/requirements/edx/testing.txt @@ -578,7 +578,7 @@ edx-drf-extensions==10.6.0 # edxval # enterprise-integrated-channels # openedx-learning -edx-enterprise==6.4.4 +edx-enterprise==6.5.0 # via # -c requirements/constraints.txt # -r requirements/edx/base.txt From 8a2c451439ec8c1b51675f0a1b023dea70f0c2c8 Mon Sep 17 00:00:00 2001 From: usamasadiq Date: Sun, 12 Oct 2025 11:10:36 +0500 Subject: [PATCH 012/351] fix: replace deprecated assertDictContainsSubset() --- .../contentstore/views/tests/test_block.py | 12 +- .../djangoapps/student/tests/test_events.py | 176 +++++++------ .../tests/specs/test_testshib.py | 9 +- .../tests/test_identityserver3.py | 6 +- .../third_party_auth/tests/test_lti.py | 25 +- common/test/utils.py | 9 + .../certificates/tests/test_events.py | 124 ++++----- lms/djangoapps/commerce/tests/test_signals.py | 3 +- .../courseware/tests/test_video_mongo.py | 29 +- .../django_comment_client/base/tests_v2.py | 11 +- .../experiments/tests/test_views.py | 3 +- lms/djangoapps/grades/tests/test_events.py | 100 +++---- .../instructor_task/tests/test_integration.py | 3 +- .../tests/test_tasks_helper.py | 247 +++++++++++------- lms/djangoapps/support/tests/test_views.py | 34 ++- .../content_libraries/tests/test_runtime.py | 41 ++- .../course_groups/tests/test_events.py | 34 +-- .../djangoapps/oauth_dispatch/tests/mixins.py | 3 +- .../oauth_dispatch/tests/test_api.py | 8 +- .../oauth_dispatch/tests/test_jwt.py | 3 +- .../user_authn/views/tests/test_auto_auth.py | 6 +- .../user_authn/views/tests/test_events.py | 43 +-- .../user_authn/views/tests/test_login.py | 3 +- .../user_authn/views/tests/test_logout.py | 21 +- .../enterprise_support/tests/test_logout.py | 3 +- .../tests/test_mixed_modulestore.py | 32 ++- 26 files changed, 562 insertions(+), 426 deletions(-) diff --git a/cms/djangoapps/contentstore/views/tests/test_block.py b/cms/djangoapps/contentstore/views/tests/test_block.py index 01aff3d613c1..e01263259bcd 100644 --- a/cms/djangoapps/contentstore/views/tests/test_block.py +++ b/cms/djangoapps/contentstore/views/tests/test_block.py @@ -86,6 +86,7 @@ add_container_page_publishing_info, create_xblock_info, ) +from common.test.utils import assert_dict_contains_subset class AsideTest(XBlockAside): @@ -863,15 +864,16 @@ def test_duplicate_event(self): XBLOCK_DUPLICATED.connect(event_receiver) usage_key = self._duplicate_and_verify(self.vert_usage_key, self.seq_usage_key) event_receiver.assert_called() - self.assertDictContainsSubset( + assert_dict_contains_subset( + self, { "signal": XBLOCK_DUPLICATED, "sender": None, "xblock_info": DuplicatedXBlockData( - usage_key=usage_key, - block_type=usage_key.block_type, - source_usage_key=self.vert_usage_key, - ), + usage_key=usage_key, + block_type=usage_key.block_type, + source_usage_key=self.vert_usage_key, + ), }, event_receiver.call_args.kwargs, ) diff --git a/common/djangoapps/student/tests/test_events.py b/common/djangoapps/student/tests/test_events.py index f336396e6e71..9d57c8ca8567 100644 --- a/common/djangoapps/student/tests/test_events.py +++ b/common/djangoapps/student/tests/test_events.py @@ -36,6 +36,7 @@ from xmodule.modulestore.tests.django_utils import \ SharedModuleStoreTestCase # lint-amnesty, pylint: disable=wrong-import-order from xmodule.modulestore.tests.factories import CourseFactory # lint-amnesty, pylint: disable=wrong-import-order +from common.test.utils import assert_dict_contains_subset class TestUserProfileEvents(UserSettingsEventTestMixin, TestCase): @@ -271,30 +272,31 @@ def test_enrollment_created_event_emitted(self): enrollment = CourseEnrollment.enroll(self.user, self.course.id) self.assertTrue(self.receiver_called) - self.assertDictContainsSubset( + assert_dict_contains_subset( + self, { "signal": COURSE_ENROLLMENT_CREATED, "sender": None, "enrollment": CourseEnrollmentData( - user=UserData( - pii=UserPersonalData( - username=self.user.username, - email=self.user.email, - name=self.user.profile.name, - ), - id=self.user.id, - is_active=self.user.is_active, - ), - course=CourseData( - course_key=self.course.id, - display_name=self.course.display_name, - ), - mode=enrollment.mode, - is_active=enrollment.is_active, - creation_date=enrollment.created, - ), + user=UserData( + pii=UserPersonalData( + username=self.user.username, + email=self.user.email, + name=self.user.profile.name, + ), + id=self.user.id, + is_active=self.user.is_active, + ), + course=CourseData( + course_key=self.course.id, + display_name=self.course.display_name, + ), + mode=enrollment.mode, + is_active=enrollment.is_active, + creation_date=enrollment.created, + ), }, - event_receiver.call_args.kwargs + event_receiver.call_args.kwargs, ) def test_enrollment_changed_event_emitted(self): @@ -314,30 +316,31 @@ def test_enrollment_changed_event_emitted(self): enrollment.update_enrollment(mode="verified") self.assertTrue(self.receiver_called) - self.assertDictContainsSubset( + assert_dict_contains_subset( + self, { "signal": COURSE_ENROLLMENT_CHANGED, "sender": None, "enrollment": CourseEnrollmentData( - user=UserData( - pii=UserPersonalData( - username=self.user.username, - email=self.user.email, - name=self.user.profile.name, - ), - id=self.user.id, - is_active=self.user.is_active, - ), - course=CourseData( - course_key=self.course.id, - display_name=self.course.display_name, - ), - mode=enrollment.mode, - is_active=enrollment.is_active, - creation_date=enrollment.created, - ), + user=UserData( + pii=UserPersonalData( + username=self.user.username, + email=self.user.email, + name=self.user.profile.name, + ), + id=self.user.id, + is_active=self.user.is_active, + ), + course=CourseData( + course_key=self.course.id, + display_name=self.course.display_name, + ), + mode=enrollment.mode, + is_active=enrollment.is_active, + creation_date=enrollment.created, + ), }, - event_receiver.call_args.kwargs + event_receiver.call_args.kwargs, ) def test_unenrollment_completed_event_emitted(self): @@ -357,30 +360,31 @@ def test_unenrollment_completed_event_emitted(self): CourseEnrollment.unenroll(self.user, self.course.id) self.assertTrue(self.receiver_called) - self.assertDictContainsSubset( + assert_dict_contains_subset( + self, { "signal": COURSE_UNENROLLMENT_COMPLETED, "sender": None, "enrollment": CourseEnrollmentData( - user=UserData( - pii=UserPersonalData( - username=self.user.username, - email=self.user.email, - name=self.user.profile.name, - ), - id=self.user.id, - is_active=self.user.is_active, - ), - course=CourseData( - course_key=self.course.id, - display_name=self.course.display_name, - ), - mode=enrollment.mode, - is_active=False, - creation_date=enrollment.created, - ), + user=UserData( + pii=UserPersonalData( + username=self.user.username, + email=self.user.email, + name=self.user.profile.name, + ), + id=self.user.id, + is_active=self.user.is_active, + ), + course=CourseData( + course_key=self.course.id, + display_name=self.course.display_name, + ), + mode=enrollment.mode, + is_active=False, + creation_date=enrollment.created, + ), }, - event_receiver.call_args.kwargs + event_receiver.call_args.kwargs, ) @@ -430,25 +434,26 @@ def test_access_role_created_event_emitted(self, AccessRole): role.add_users(self.user) self.assertTrue(self.receiver_called) - self.assertDictContainsSubset( + assert_dict_contains_subset( + self, { "signal": COURSE_ACCESS_ROLE_ADDED, "sender": None, "course_access_role_data": CourseAccessRoleData( - user=UserData( - pii=UserPersonalData( - username=self.user.username, - email=self.user.email, - ), - id=self.user.id, - is_active=self.user.is_active, - ), - course_key=self.course_key, - org_key=self.course_key.org, - role=role._role_name, # pylint: disable=protected-access - ), + user=UserData( + pii=UserPersonalData( + username=self.user.username, + email=self.user.email, + ), + id=self.user.id, + is_active=self.user.is_active, + ), + course_key=self.course_key, + org_key=self.course_key.org, + role=role._role_name, # pylint: disable=protected-access + ), }, - event_receiver.call_args.kwargs + event_receiver.call_args.kwargs, ) @ddt.data( @@ -468,23 +473,24 @@ def test_access_role_removed_event_emitted(self, AccessRole): role.remove_users(self.user) self.assertTrue(self.receiver_called) - self.assertDictContainsSubset( + assert_dict_contains_subset( + self, { "signal": COURSE_ACCESS_ROLE_REMOVED, "sender": None, "course_access_role_data": CourseAccessRoleData( - user=UserData( - pii=UserPersonalData( - username=self.user.username, - email=self.user.email, - ), - id=self.user.id, - is_active=self.user.is_active, - ), - course_key=self.course_key, - org_key=self.course_key.org, - role=role._role_name, # pylint: disable=protected-access - ), + user=UserData( + pii=UserPersonalData( + username=self.user.username, + email=self.user.email, + ), + id=self.user.id, + is_active=self.user.is_active, + ), + course_key=self.course_key, + org_key=self.course_key.org, + role=role._role_name, # pylint: disable=protected-access + ), }, - event_receiver.call_args.kwargs + event_receiver.call_args.kwargs, ) diff --git a/common/djangoapps/third_party_auth/tests/specs/test_testshib.py b/common/djangoapps/third_party_auth/tests/specs/test_testshib.py index caddd325ba76..18059ac6873c 100644 --- a/common/djangoapps/third_party_auth/tests/specs/test_testshib.py +++ b/common/djangoapps/third_party_auth/tests/specs/test_testshib.py @@ -30,6 +30,7 @@ from openedx.features.enterprise_support.tests.factories import EnterpriseCustomerFactory from .base import IntegrationTestMixin +from common.test.utils import assert_dict_contains_subset TESTSHIB_ENTITY_ID = "https://idp.testshib.org/idp/shibboleth" TESTSHIB_METADATA_URL = "https://mock.testshib.org/metadata/testshib-providers.xml" @@ -402,8 +403,10 @@ def test_debug_mode_login(self, debug_mode_enabled): assert msg.startswith("SAML login %s") assert action_type == "request" assert idp_name == self.PROVIDER_IDP_SLUG - self.assertDictContainsSubset( - {"idp": idp_name, "auth_entry": "login", "next": expected_next_url}, request_data + assert_dict_contains_subset( + self, + {"idp": idp_name, "auth_entry": "login", "next": expected_next_url}, + request_data, ) assert next_url == expected_next_url assert " Section > Subsection > Problem1', - 'block_key': 'block-v1:edx+1.23x+test_course+type@problem+block@Problem1', - 'title': 'Problem1', - }, student_data[0]) + assert_dict_contains_subset( + self, + { + 'username': 'student', + 'location': 'test_course > Section > Subsection > Problem1', + 'block_key': 'block-v1:edx+1.23x+test_course+type@problem+block@Problem1', + 'title': 'Problem1', + }, + student_data[0], + ) assert 'state' in student_data[0] assert student_data_keys_list == ['username', 'title', 'location', 'block_key', 'state'] mock_list_problem_responses.assert_called_with(self.course.id, ANY, ANY) @@ -569,22 +576,30 @@ def test_build_student_data_for_block_with_mock_generate_report_data(self, mock_ usage_key_str_list=[str(self.course.location)], ) assert len(student_data) == 2 - self.assertDictContainsSubset({ - 'username': 'student', - 'location': 'test_course > Section > Subsection > Problem1', - 'block_key': 'block-v1:edx+1.23x+test_course+type@problem+block@Problem1', - 'title': 'Problem1', - 'some': 'state1', - 'more': 'state1!', - }, student_data[0]) - self.assertDictContainsSubset({ - 'username': 'student', - 'location': 'test_course > Section > Subsection > Problem1', - 'block_key': 'block-v1:edx+1.23x+test_course+type@problem+block@Problem1', - 'title': 'Problem1', - 'some': 'state2', - 'more': 'state2!', - }, student_data[1]) + assert_dict_contains_subset( + self, + { + 'username': 'student', + 'location': 'test_course > Section > Subsection > Problem1', + 'block_key': 'block-v1:edx+1.23x+test_course+type@problem+block@Problem1', + 'title': 'Problem1', + 'some': 'state1', + 'more': 'state1!', + }, + student_data[0], + ) + assert_dict_contains_subset( + self, + { + 'username': 'student', + 'location': 'test_course > Section > Subsection > Problem1', + 'block_key': 'block-v1:edx+1.23x+test_course+type@problem+block@Problem1', + 'title': 'Problem1', + 'some': 'state2', + 'more': 'state2!', + }, + student_data[1], + ) assert student_data[0]['state'] == student_data[1]['state'] assert student_data_keys_list == ['username', 'title', 'location', 'more', 'some', 'block_key', 'state'] @@ -610,22 +625,30 @@ def test_build_student_data_for_block_with_ordered_generate_report_data(self, mo usage_key_str_list=[str(self.course.location)], ) assert len(student_data) == 2 - self.assertDictContainsSubset({ - 'username': 'student', - 'location': 'test_course > Section > Subsection > Problem1', - 'block_key': 'block-v1:edx+1.23x+test_course+type@problem+block@Problem1', - 'title': 'Problem1', - 'some': 'state1', - 'more': 'state1!', - }, student_data[0]) - self.assertDictContainsSubset({ - 'username': 'student', - 'location': 'test_course > Section > Subsection > Problem1', - 'block_key': 'block-v1:edx+1.23x+test_course+type@problem+block@Problem1', - 'title': 'Problem1', - 'some': 'state2', - 'more': 'state2!', - }, student_data[1]) + assert_dict_contains_subset( + self, + { + 'username': 'student', + 'location': 'test_course > Section > Subsection > Problem1', + 'block_key': 'block-v1:edx+1.23x+test_course+type@problem+block@Problem1', + 'title': 'Problem1', + 'some': 'state1', + 'more': 'state1!', + }, + student_data[0], + ) + assert_dict_contains_subset( + self, + { + 'username': 'student', + 'location': 'test_course > Section > Subsection > Problem1', + 'block_key': 'block-v1:edx+1.23x+test_course+type@problem+block@Problem1', + 'title': 'Problem1', + 'some': 'state2', + 'more': 'state2!', + }, + student_data[1], + ) assert student_data[0]['state'] == student_data[1]['state'] assert student_data_keys_list == ['username', 'title', 'location', 'some', 'more', 'block_key', 'state'] @@ -642,16 +665,20 @@ def test_build_student_data_for_block_with_real_generate_report_data(self): usage_key_str_list=[str(self.course.location)], ) assert len(student_data) == 1 - self.assertDictContainsSubset({ - 'username': 'student', - 'location': 'test_course > Section > Subsection > Problem1', - 'block_key': 'block-v1:edx+1.23x+test_course+type@problem+block@Problem1', - 'title': 'Problem1', - 'Answer ID': 'Problem1_2_1', - 'Answer': 'Option 1', - 'Correct Answer': 'Option 1', - 'Question': 'The correct answer is Option 1', - }, student_data[0]) + assert_dict_contains_subset( + self, + { + 'username': 'student', + 'location': 'test_course > Section > Subsection > Problem1', + 'block_key': 'block-v1:edx+1.23x+test_course+type@problem+block@Problem1', + 'title': 'Problem1', + 'Answer ID': 'Problem1_2_1', + 'Answer': 'Option 1', + 'Correct Answer': 'Option 1', + 'Question': 'The correct answer is Option 1', + }, + student_data[0], + ) assert 'state' in student_data[0] assert student_data_keys_list == ['username', 'title', 'location', 'Answer', 'Answer ID', 'Correct Answer', 'Question', 'block_key', 'state'] @@ -671,16 +698,20 @@ def test_build_student_data_for_multiple_problems(self): ) assert len(student_data) == 2 for idx in range(1, 3): - self.assertDictContainsSubset({ - 'username': 'student', - 'location': f'test_course > Section > Subsection > Problem{idx}', - 'block_key': f'block-v1:edx+1.23x+test_course+type@problem+block@Problem{idx}', - 'title': f'Problem{idx}', - 'Answer ID': f'Problem{idx}_2_1', - 'Answer': 'Option 1', - 'Correct Answer': 'Option 1', - 'Question': 'The correct answer is Option 1', - }, student_data[idx - 1]) + assert_dict_contains_subset( + self, + { + 'username': 'student', + 'location': f'test_course > Section > Subsection > Problem{idx}', + 'block_key': f'block-v1:edx+1.23x+test_course+type@problem+block@Problem{idx}', + 'title': f'Problem{idx}', + 'Answer ID': f'Problem{idx}_2_1', + 'Answer': 'Option 1', + 'Correct Answer': 'Option 1', + 'Question': 'The correct answer is Option 1', + }, + student_data[idx - 1], + ) assert 'state' in student_data[(idx - 1)] @ddt.data( @@ -819,7 +850,7 @@ def test_no_problems(self, use_tempfile, _): """ with patch(USE_ON_DISK_GRADE_REPORT, return_value=use_tempfile): result = ProblemGradeReport.generate(None, None, self.course.id, {}, 'graded') - self.assertDictContainsSubset({'action_name': 'graded', 'attempted': 2, 'succeeded': 2, 'failed': 0}, result) + assert_dict_contains_subset(self, {'action_name': 'graded', 'attempted': 2, 'succeeded': 2, 'failed': 0}, result) self.verify_rows_in_csv([ dict(list(zip( self.csv_header_row, @@ -845,7 +876,7 @@ def test_single_problem(self, use_tempfile, _): self.submit_student_answer(self.student_1.username, 'Problem1', ['Option 1']) with patch(USE_ON_DISK_GRADE_REPORT, return_value=use_tempfile): result = ProblemGradeReport.generate(None, None, self.course.id, {}, 'graded') - self.assertDictContainsSubset({'action_name': 'graded', 'attempted': 2, 'succeeded': 2, 'failed': 0}, result) + assert_dict_contains_subset(self, {'action_name': 'graded', 'attempted': 2, 'succeeded': 2, 'failed': 0}, result) problem_name = 'Homework 1: Subsection - Problem1' header_row = self.csv_header_row + [problem_name + ' (Earned)', problem_name + ' (Possible)'] self.verify_rows_in_csv([ @@ -891,8 +922,10 @@ def test_single_problem_verified_student_only(self, use_tempfile, _): self.submit_student_answer(student_verified.username, 'Problem1', ['Option 1']) with patch(USE_ON_DISK_GRADE_REPORT, return_value=use_tempfile): result = ProblemGradeReport.generate(None, None, self.course.id, {}, 'graded') - self.assertDictContainsSubset( - {'action_name': 'graded', 'attempted': 1, 'succeeded': 1, 'failed': 0}, result + assert_dict_contains_subset( + self, + {'action_name': 'graded', 'attempted': 1, 'succeeded': 1, 'failed': 0}, + result, ) @patch('lms.djangoapps.instructor_task.tasks_helper.runner._get_current_task') @@ -913,7 +946,7 @@ def test_inactive_enrollment_included(self, use_tempfile, _): self.submit_student_answer(self.student_1.username, 'Problem1', ['Option 1']) with patch(USE_ON_DISK_GRADE_REPORT, return_value=use_tempfile): result = ProblemGradeReport.generate(None, None, self.course.id, {}, 'graded') - self.assertDictContainsSubset({'action_name': 'graded', 'attempted': 3, 'succeeded': 3, 'failed': 0}, result) + assert_dict_contains_subset(self, {'action_name': 'graded', 'attempted': 3, 'succeeded': 3, 'failed': 0}, result) problem_name = 'Homework 1: Subsection - Problem1' header_row = self.csv_header_row + [problem_name + ' (Earned)', problem_name + ' (Possible)'] self.verify_rows_in_csv([ @@ -987,8 +1020,10 @@ def test_problem_grade_report(self, use_tempfile): with patch('lms.djangoapps.instructor_task.tasks_helper.runner._get_current_task'): with patch(USE_ON_DISK_GRADE_REPORT, return_value=use_tempfile): result = ProblemGradeReport.generate(None, None, self.course.id, {}, 'graded') - self.assertDictContainsSubset( - {'action_name': 'graded', 'attempted': 2, 'succeeded': 2, 'failed': 0}, result + assert_dict_contains_subset( + self, + {'action_name': 'graded', 'attempted': 2, 'succeeded': 2, 'failed': 0}, + result, ) problem_names = ['Homework 1: Subsection - problem_a_url', 'Homework 1: Subsection - problem_b_url'] @@ -1143,8 +1178,10 @@ def test_cohort_content(self, use_tempfile): with patch('lms.djangoapps.instructor_task.tasks_helper.runner._get_current_task'): with patch(USE_ON_DISK_GRADE_REPORT, return_value=use_tempfile): result = ProblemGradeReport.generate(None, None, self.course.id, {}, 'graded') - self.assertDictContainsSubset( - {'action_name': 'graded', 'attempted': 5, 'succeeded': 5, 'failed': 0}, result + assert_dict_contains_subset( + self, + {'action_name': 'graded', 'attempted': 5, 'succeeded': 5, 'failed': 0}, + result, ) problem_names = ['Homework 1: Subsection - Problem0', 'Homework 1: Subsection - Problem1'] header_row = ['Student ID', 'Email', 'Username', 'Enrollment Status', 'Grade'] @@ -1228,7 +1265,7 @@ def test_successfully_generate_course_survey_report(self): None, None, self.course.id, task_input, 'generating course survey report' ) - self.assertDictContainsSubset({'attempted': 2, 'succeeded': 2, 'failed': 0}, result) + assert_dict_contains_subset(self, {'attempted': 2, 'succeeded': 2, 'failed': 0}, result) def test_generate_course_survey_report(self): """ @@ -1262,7 +1299,7 @@ def test_generate_course_survey_report(self): ]) expected_data = [header_row, student1_row, student2_row] - self.assertDictContainsSubset({'attempted': 2, 'succeeded': 2, 'failed': 0}, result) + assert_dict_contains_subset(self, {'attempted': 2, 'succeeded': 2, 'failed': 0}, result) self._verify_csv_file_report(report_store, expected_data) def _verify_csv_file_report(self, report_store, expected_data): @@ -1297,7 +1334,7 @@ def test_success(self): links = report_store.links_for(self.course.id) assert len(links) == 1 - self.assertDictContainsSubset({'attempted': 1, 'succeeded': 1, 'failed': 0}, result) + assert_dict_contains_subset(self, {'attempted': 1, 'succeeded': 1, 'failed': 0}, result) def test_custom_directory(self): self.create_student('student', 'student@example.com') @@ -1352,7 +1389,7 @@ def test_unicode_usernames(self, students): result = upload_students_csv(None, None, self.course.id, task_input, 'calculated') # This assertion simply confirms that the generation completed with no errors num_students = len(students) - self.assertDictContainsSubset({'attempted': num_students, 'succeeded': num_students, 'failed': 0}, result) + assert_dict_contains_subset(self, {'attempted': num_students, 'succeeded': num_students, 'failed': 0}, result) class TestTeamStudentReport(TestReportMixin, InstructorTaskCourseTestCase): @@ -1382,7 +1419,7 @@ def _generate_and_verify_teams_column(self, username, expected_team): with patch('lms.djangoapps.instructor_task.tasks_helper.runner._get_current_task') as mock_current_task: mock_current_task.return_value = current_task result = upload_students_csv(None, None, self.course.id, task_input, 'calculated') - self.assertDictContainsSubset({'attempted': 2, 'succeeded': 2, 'failed': 0}, result) + assert_dict_contains_subset(self, {'attempted': 2, 'succeeded': 2, 'failed': 0}, result) report_store = ReportStore.from_config(config_name='GRADES_DOWNLOAD') report_csv_filename = report_store.links_for(self.course.id)[0][0] report_path = report_store.path_to(self.course.id, report_csv_filename) @@ -1447,7 +1484,7 @@ def test_success(self): links = report_store.links_for(self.course.id) assert len(links) == 1 - self.assertDictContainsSubset({'attempted': 1, 'succeeded': 1, 'failed': 0}, result) + assert_dict_contains_subset(self, {'attempted': 1, 'succeeded': 1, 'failed': 0}, result) def test_unicode_email_addresses(self): """ @@ -1463,7 +1500,7 @@ def test_unicode_email_addresses(self): result = upload_may_enroll_csv(None, None, self.course.id, task_input, 'calculated') # This assertion simply confirms that the generation completed with no errors num_enrollments = len(enrollments) - self.assertDictContainsSubset({'attempted': num_enrollments, 'succeeded': num_enrollments, 'failed': 0}, result) + assert_dict_contains_subset(self, {'attempted': num_enrollments, 'succeeded': num_enrollments, 'failed': 0}, result) class MockDefaultStorage: @@ -1510,7 +1547,7 @@ def test_username(self): 'student_1\xec,,Cohort 1\n' 'student_2,,Cohort 2' ) - self.assertDictContainsSubset({'total': 2, 'attempted': 2, 'succeeded': 2, 'failed': 0}, result) + assert_dict_contains_subset(self, {'total': 2, 'attempted': 2, 'succeeded': 2, 'failed': 0}, result) self.verify_rows_in_csv( [ dict(list(zip(self.csv_header_row, ['Cohort 1', 'True', '1', '', '', '']))), @@ -1525,7 +1562,7 @@ def test_email(self): ',student_1@example.com,Cohort 1\n' ',student_2@example.com,Cohort 2' ) - self.assertDictContainsSubset({'total': 2, 'attempted': 2, 'succeeded': 2, 'failed': 0}, result) + assert_dict_contains_subset(self, {'total': 2, 'attempted': 2, 'succeeded': 2, 'failed': 0}, result) self.verify_rows_in_csv( [ dict(list(zip(self.csv_header_row, ['Cohort 1', 'True', '1', '', '', '']))), @@ -1540,7 +1577,7 @@ def test_username_and_email(self): 'student_1\xec,student_1@example.com,Cohort 1\n' 'student_2,student_2@example.com,Cohort 2' ) - self.assertDictContainsSubset({'total': 2, 'attempted': 2, 'succeeded': 2, 'failed': 0}, result) + assert_dict_contains_subset(self, {'total': 2, 'attempted': 2, 'succeeded': 2, 'failed': 0}, result) self.verify_rows_in_csv( [ dict(list(zip(self.csv_header_row, ['Cohort 1', 'True', '1', '', '', '']))), @@ -1561,7 +1598,7 @@ def test_prefer_email(self): 'student_1\xec,student_1@example.com,Cohort 1\n' # valid username and email 'Invalid,student_2@example.com,Cohort 2' # invalid username, valid email ) - self.assertDictContainsSubset({'total': 2, 'attempted': 2, 'succeeded': 2, 'failed': 0}, result) + assert_dict_contains_subset(self, {'total': 2, 'attempted': 2, 'succeeded': 2, 'failed': 0}, result) self.verify_rows_in_csv( [ dict(list(zip(self.csv_header_row, ['Cohort 1', 'True', '1', '', '', '']))), @@ -1575,7 +1612,7 @@ def test_non_existent_user(self): 'username,email,cohort\n' 'Invalid,,Cohort 1\n' ) - self.assertDictContainsSubset({'total': 1, 'attempted': 1, 'succeeded': 0, 'failed': 1}, result) + assert_dict_contains_subset(self, {'total': 1, 'attempted': 1, 'succeeded': 0, 'failed': 1}, result) self.verify_rows_in_csv( [ dict(list(zip(self.csv_header_row, ['Cohort 1', 'True', '0', 'Invalid', '', '']))), @@ -1589,7 +1626,7 @@ def test_non_existent_cohort(self): ',student_1@example.com,Does Not Exist\n' 'student_2,,Cohort 2' ) - self.assertDictContainsSubset({'total': 2, 'attempted': 2, 'succeeded': 1, 'failed': 1}, result) + assert_dict_contains_subset(self, {'total': 2, 'attempted': 2, 'succeeded': 1, 'failed': 1}, result) self.verify_rows_in_csv( [ dict(list(zip(self.csv_header_row, ['Does Not Exist', 'False', '0', '', '', '']))), @@ -1603,8 +1640,11 @@ def test_preassigned_user(self): 'username,email,cohort\n' ',example_email@example.com,Cohort 1' ) - self.assertDictContainsSubset({'total': 1, 'attempted': 1, 'succeeded': 0, 'failed': 0}, - result) + assert_dict_contains_subset( + self, + {'total': 1, 'attempted': 1, 'succeeded': 0, 'failed': 0}, + result, + ) self.verify_rows_in_csv( [ dict(list(zip(self.csv_header_row, ['Cohort 1', 'True', '0', '', '', 'example_email@example.com']))), @@ -1617,7 +1657,7 @@ def test_invalid_email(self): 'username,email,cohort\n' ',student_1@,Cohort 1\n' ) - self.assertDictContainsSubset({'total': 1, 'attempted': 1, 'succeeded': 0, 'failed': 1}, result) + assert_dict_contains_subset(self, {'total': 1, 'attempted': 1, 'succeeded': 0, 'failed': 1}, result) self.verify_rows_in_csv( [ dict(list(zip(self.csv_header_row, ['Cohort 1', 'True', '0', '', 'student_1@', '']))), @@ -1642,7 +1682,7 @@ def test_too_few_commas(self): 'student_1\xec,\n' 'student_2' ) - self.assertDictContainsSubset({'total': 2, 'attempted': 2, 'succeeded': 0, 'failed': 2}, result) + assert_dict_contains_subset(self, {'total': 2, 'attempted': 2, 'succeeded': 0, 'failed': 2}, result) self.verify_rows_in_csv( [ dict(list(zip(self.csv_header_row, ['', 'False', '0', '', '', '']))), @@ -1654,7 +1694,7 @@ def test_only_header_row(self): result = self._cohort_students_and_upload( 'username,email,cohort' ) - self.assertDictContainsSubset({'total': 0, 'attempted': 0, 'succeeded': 0, 'failed': 0}, result) + assert_dict_contains_subset(self, {'total': 0, 'attempted': 0, 'succeeded': 0, 'failed': 0}, result) self.verify_rows_in_csv([]) def test_carriage_return(self): @@ -1666,7 +1706,7 @@ def test_carriage_return(self): 'student_1\xec,,Cohort 1\r' 'student_2,,Cohort 2' ) - self.assertDictContainsSubset({'total': 2, 'attempted': 2, 'succeeded': 2, 'failed': 0}, result) + assert_dict_contains_subset(self, {'total': 2, 'attempted': 2, 'succeeded': 2, 'failed': 0}, result) self.verify_rows_in_csv( [ dict(list(zip(self.csv_header_row, ['Cohort 1', 'True', '1', '', '', '']))), @@ -1684,7 +1724,7 @@ def test_carriage_return_line_feed(self): 'student_1\xec,,Cohort 1\r\n' 'student_2,,Cohort 2' ) - self.assertDictContainsSubset({'total': 2, 'attempted': 2, 'succeeded': 2, 'failed': 0}, result) + assert_dict_contains_subset(self, {'total': 2, 'attempted': 2, 'succeeded': 2, 'failed': 0}, result) self.verify_rows_in_csv( [ dict(list(zip(self.csv_header_row, ['Cohort 1', 'True', '1', '', '', '']))), @@ -1704,7 +1744,7 @@ def test_move_users_to_new_cohort(self): 'student_1\xec,,Cohort 2\n' 'student_2,,Cohort 1' ) - self.assertDictContainsSubset({'total': 2, 'attempted': 2, 'succeeded': 2, 'failed': 0}, result) + assert_dict_contains_subset(self, {'total': 2, 'attempted': 2, 'succeeded': 2, 'failed': 0}, result) self.verify_rows_in_csv( [ dict(list(zip(self.csv_header_row, ['Cohort 1', 'True', '1', '', '', '']))), @@ -1724,7 +1764,7 @@ def test_move_users_to_same_cohort(self): 'student_1\xec,,Cohort 1\n' 'student_2,,Cohort 2' ) - self.assertDictContainsSubset({'total': 2, 'attempted': 2, 'skipped': 2, 'failed': 0}, result) + assert_dict_contains_subset(self, {'total': 2, 'attempted': 2, 'skipped': 2, 'failed': 0}, result) self.verify_rows_in_csv( [ dict(list(zip(self.csv_header_row, ['Cohort 1', 'True', '0', '', '', '']))), @@ -1809,7 +1849,8 @@ def test_grade_report(self): with patch('lms.djangoapps.instructor_task.tasks_helper.runner._get_current_task'): result = CourseGradeReport.generate(None, None, self.course.id, {}, 'graded') - self.assertDictContainsSubset( + assert_dict_contains_subset( + self, {'action_name': 'graded', 'attempted': 1, 'succeeded': 1, 'failed': 0}, result, ) @@ -1868,7 +1909,8 @@ def test_grade_report_with_overrides(self): with patch('lms.djangoapps.instructor_task.tasks_helper.runner._get_current_task'): result = CourseGradeReport.generate(None, None, self.course.id, {}, 'graded') - self.assertDictContainsSubset( + assert_dict_contains_subset( + self, {'action_name': 'graded', 'attempted': 1, 'succeeded': 1, 'failed': 0}, result, ) @@ -1912,8 +1954,10 @@ def test_course_grade_with_verified_student_only(self, _get_current_task): self.submit_student_answer(student_1.username, 'Problem4', ['Option 1']) self.submit_student_answer(student_verified.username, 'Problem4', ['Option 1']) result = CourseGradeReport.generate(None, None, self.course.id, {}, 'graded') - self.assertDictContainsSubset( - {'action_name': 'graded', 'attempted': 1, 'succeeded': 1, 'failed': 0}, result + assert_dict_contains_subset( + self, + {'action_name': 'graded', 'attempted': 1, 'succeeded': 1, 'failed': 0}, + result, ) @ddt.data(True, False) @@ -2517,9 +2561,10 @@ def assertCertificatesGenerated(self, task_input, expected_results): None, None, self.course.id, task_input, 'certificates generated' ) - self.assertDictContainsSubset( + assert_dict_contains_subset( + self, expected_results, - result + result, ) def _create_students(self, number_of_students): diff --git a/lms/djangoapps/support/tests/test_views.py b/lms/djangoapps/support/tests/test_views.py index 4abc4508ea35..df713daf0b98 100644 --- a/lms/djangoapps/support/tests/test_views.py +++ b/lms/djangoapps/support/tests/test_views.py @@ -52,7 +52,7 @@ UserFactory, ) from common.djangoapps.third_party_auth.tests.factories import SAMLProviderConfigFactory -from common.test.utils import disable_signal +from common.test.utils import disable_signal, assert_dict_contains_subset from lms.djangoapps.program_enrollments.tests.factories import ProgramCourseEnrollmentFactory, ProgramEnrollmentFactory from lms.djangoapps.support.models import CourseResetAudit from lms.djangoapps.support.serializers import ProgramEnrollmentSerializer @@ -343,14 +343,18 @@ def test_get_enrollments(self, search_string_type): assert response.status_code == 200 data = json.loads(response.content.decode('utf-8')) assert len(data) == 1 - self.assertDictContainsSubset({ - 'mode': CourseMode.AUDIT, - 'manual_enrollment': {}, - 'user': self.student.username, - 'course_id': str(self.course.id), - 'is_active': True, - 'verified_upgrade_deadline': None, - }, data[0]) + assert_dict_contains_subset( + self, + { + 'mode': CourseMode.AUDIT, + 'manual_enrollment': {}, + 'user': self.student.username, + 'course_id': str(self.course.id), + 'is_active': True, + 'verified_upgrade_deadline': None, + }, + data[0], + ) assert {CourseMode.VERIFIED, CourseMode.AUDIT, CourseMode.HONOR, CourseMode.NO_ID_PROFESSIONAL_MODE, CourseMode.PROFESSIONAL, CourseMode.CREDIT_MODE} == {mode['slug'] for mode in data[0]['course_modes']} assert 'enterprise_course_enrollments' not in data[0] @@ -471,10 +475,14 @@ def test_get_manual_enrollment_history(self): ) response = self.client.get(self.url) assert response.status_code == 200 - self.assertDictContainsSubset({ - 'enrolled_by': self.user.email, - 'reason': 'Financial Assistance', - }, json.loads(response.content.decode('utf-8'))[0]['manual_enrollment']) + assert_dict_contains_subset( + self, + { + 'enrolled_by': self.user.email, + 'reason': 'Financial Assistance', + }, + json.loads(response.content.decode('utf-8'))[0]['manual_enrollment'], + ) @disable_signal(signals, 'post_save') @ddt.data('username', 'email') diff --git a/openedx/core/djangoapps/content_libraries/tests/test_runtime.py b/openedx/core/djangoapps/content_libraries/tests/test_runtime.py index 9cb10e514a60..97699dbf8d6b 100644 --- a/openedx/core/djangoapps/content_libraries/tests/test_runtime.py +++ b/openedx/core/djangoapps/content_libraries/tests/test_runtime.py @@ -27,6 +27,7 @@ from openedx.core.djangolib.testing.utils import skip_unless_lms, skip_unless_cms from openedx.core.lib.xblock_serializer import api as serializer_api from common.djangoapps.student.tests.factories import UserFactory +from common.test.utils import assert_dict_contains_subset class ContentLibraryContentTestMixin: @@ -205,10 +206,14 @@ def test_xblock_metadata(self): assert metadata_view_result.data['display_name'] == 'New Multi Choice Question' assert 'children' not in metadata_view_result.data assert 'editable_children' not in metadata_view_result.data - self.assertDictContainsSubset({ - "content_type": "CAPA", - "problem_types": ["multiplechoiceresponse"], - }, metadata_view_result.data["index_dictionary"]) + assert_dict_contains_subset( + self, + { + "content_type": "CAPA", + "problem_types": ["multiplechoiceresponse"], + }, + metadata_view_result.data["index_dictionary"], + ) assert metadata_view_result.data['student_view_data'] is None # Capa doesn't provide student_view_data @@ -493,11 +498,15 @@ def test_scores_persisted(self): submit_result = client.post(problem_check_url, data={problem_key: "choice_3"}) assert submit_result.status_code == 200 submit_data = json.loads(submit_result.content.decode('utf-8')) - self.assertDictContainsSubset({ - "current_score": 0, - "total_possible": 1, - "attempts_used": 1, - }, submit_data) + assert_dict_contains_subset( + self, + { + "current_score": 0, + "total_possible": 1, + "attempts_used": 1, + }, + submit_data, + ) # Now test that the score is also persisted in StudentModule: # If we add a REST API to get an individual block's score, that should be checked instead of StudentModule. @@ -509,11 +518,15 @@ def test_scores_persisted(self): submit_result = client.post(problem_check_url, data={problem_key: "choice_1"}) assert submit_result.status_code == 200 submit_data = json.loads(submit_result.content.decode('utf-8')) - self.assertDictContainsSubset({ - "current_score": 1, - "total_possible": 1, - "attempts_used": 2, - }, submit_data) + assert_dict_contains_subset( + self, + { + "current_score": 1, + "total_possible": 1, + "attempts_used": 2, + }, + submit_data, + ) # Now test that the score is also updated in StudentModule: # If we add a REST API to get an individual block's score, that should be checked instead of StudentModule. sm = get_score(self.student_a, block_id) diff --git a/openedx/core/djangoapps/course_groups/tests/test_events.py b/openedx/core/djangoapps/course_groups/tests/test_events.py index 616a7bb3f156..40a04dea8c8b 100644 --- a/openedx/core/djangoapps/course_groups/tests/test_events.py +++ b/openedx/core/djangoapps/course_groups/tests/test_events.py @@ -18,6 +18,7 @@ from openedx.core.djangoapps.course_groups.tests.helpers import CohortFactory from xmodule.modulestore.tests.django_utils import SharedModuleStoreTestCase # lint-amnesty, pylint: disable=wrong-import-order +from common.test.utils import assert_dict_contains_subset @skip_unless_lms @@ -90,25 +91,26 @@ def test_send_cohort_membership_changed_event(self): ) self.assertTrue(self.receiver_called) - self.assertDictContainsSubset( + assert_dict_contains_subset( + self, { "signal": COHORT_MEMBERSHIP_CHANGED, "sender": None, "cohort": CohortData( - user=UserData( - pii=UserPersonalData( - username=cohort_membership.user.username, - email=cohort_membership.user.email, - name=cohort_membership.user.profile.name, - ), - id=cohort_membership.user.id, - is_active=cohort_membership.user.is_active, - ), - course=CourseData( - course_key=cohort_membership.course_id, - ), - name=cohort_membership.course_user_group.name, - ), + user=UserData( + pii=UserPersonalData( + username=cohort_membership.user.username, + email=cohort_membership.user.email, + name=cohort_membership.user.profile.name, + ), + id=cohort_membership.user.id, + is_active=cohort_membership.user.is_active, + ), + course=CourseData( + course_key=cohort_membership.course_id, + ), + name=cohort_membership.course_user_group.name, + ), }, - event_receiver.call_args.kwargs + event_receiver.call_args.kwargs, ) diff --git a/openedx/core/djangoapps/oauth_dispatch/tests/mixins.py b/openedx/core/djangoapps/oauth_dispatch/tests/mixins.py index d99ac4883b18..407a9aac2b84 100644 --- a/openedx/core/djangoapps/oauth_dispatch/tests/mixins.py +++ b/openedx/core/djangoapps/oauth_dispatch/tests/mixins.py @@ -11,6 +11,7 @@ from jwt.exceptions import ExpiredSignatureError from common.djangoapps.student.models import UserProfile, anonymous_id_for_user +from common.test.utils import assert_dict_contains_subset class AccessTokenMixin: @@ -88,7 +89,7 @@ def _decode_jwt(verify_expiration): expected['grant_type'] = grant_type or '' - self.assertDictContainsSubset(expected, payload) + assert_dict_contains_subset(self, expected, payload) if expires_in: assert payload['exp'] == payload['iat'] + expires_in diff --git a/openedx/core/djangoapps/oauth_dispatch/tests/test_api.py b/openedx/core/djangoapps/oauth_dispatch/tests/test_api.py index 3c064cc63c55..5bf1f524bce6 100644 --- a/openedx/core/djangoapps/oauth_dispatch/tests/test_api.py +++ b/openedx/core/djangoapps/oauth_dispatch/tests/test_api.py @@ -9,6 +9,7 @@ from oauth2_provider.models import AccessToken from common.djangoapps.student.tests.factories import UserFactory +from common.test.utils import assert_dict_contains_subset OAUTH_PROVIDER_ENABLED = settings.FEATURES.get('ENABLE_OAUTH2_PROVIDER') if OAUTH_PROVIDER_ENABLED: @@ -43,7 +44,8 @@ def test_create_token_success(self): token = api.create_dot_access_token(HttpRequest(), self.user, self.client) assert token['access_token'] assert token['refresh_token'] - self.assertDictContainsSubset( + assert_dict_contains_subset( + self, { 'token_type': 'Bearer', 'expires_in': EXPECTED_DEFAULT_EXPIRES_IN, @@ -63,5 +65,5 @@ def test_create_token_overrides(self): token = api.create_dot_access_token( HttpRequest(), self.user, self.client, expires_in=expires_in, scopes=['profile'], ) - self.assertDictContainsSubset({'scope': 'profile'}, token) - self.assertDictContainsSubset({'expires_in': expires_in}, token) + assert_dict_contains_subset(self, {'scope': 'profile'}, token) + assert_dict_contains_subset(self, {'expires_in': expires_in}, token) diff --git a/openedx/core/djangoapps/oauth_dispatch/tests/test_jwt.py b/openedx/core/djangoapps/oauth_dispatch/tests/test_jwt.py index da95fd072ded..647da14f6edd 100644 --- a/openedx/core/djangoapps/oauth_dispatch/tests/test_jwt.py +++ b/openedx/core/djangoapps/oauth_dispatch/tests/test_jwt.py @@ -12,6 +12,7 @@ from openedx.core.djangoapps.oauth_dispatch.models import RestrictedApplication from openedx.core.djangoapps.oauth_dispatch.tests.mixins import AccessTokenMixin from common.djangoapps.student.tests.factories import UserFactory +from common.test.utils import assert_dict_contains_subset @ddt.ddt @@ -171,7 +172,7 @@ def test_create_jwt_for_user(self, user_email_verified, mock_create_roles): token_payload = self.assert_valid_jwt_access_token( jwt_token, self.user, self.default_scopes, aud=aud, secret=secret, ) - self.assertDictContainsSubset(additional_claims, token_payload) + assert_dict_contains_subset(self, additional_claims, token_payload) assert user_email_verified == token_payload['email_verified'] assert token_payload['roles'] == mock_create_roles.return_value diff --git a/openedx/core/djangoapps/user_authn/views/tests/test_auto_auth.py b/openedx/core/djangoapps/user_authn/views/tests/test_auto_auth.py index 0346aee25ae6..b68774903092 100644 --- a/openedx/core/djangoapps/user_authn/views/tests/test_auto_auth.py +++ b/openedx/core/djangoapps/user_authn/views/tests/test_auto_auth.py @@ -21,6 +21,7 @@ ) from openedx.core.djangoapps.django_comment_common.utils import seed_permissions_roles from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase # lint-amnesty, pylint: disable=wrong-import-order +from common.test.utils import assert_dict_contains_subset class AutoAuthTestCase(UrlResetMixin, TestCase): @@ -182,12 +183,13 @@ def test_json_response(self): for key in ['created_status', 'username', 'email', 'password', 'user_id', 'anonymous_id']: assert key in response_data user = User.objects.get(username=response_data['username']) - self.assertDictContainsSubset( + assert_dict_contains_subset( + self, { 'created_status': 'Logged in', 'anonymous_id': anonymous_id_for_user(user, None), }, - response_data + response_data, ) @ddt.data(*COURSE_IDS_DDT) diff --git a/openedx/core/djangoapps/user_authn/views/tests/test_events.py b/openedx/core/djangoapps/user_authn/views/tests/test_events.py index acec4a935d60..4460e8c627b4 100644 --- a/openedx/core/djangoapps/user_authn/views/tests/test_events.py +++ b/openedx/core/djangoapps/user_authn/views/tests/test_events.py @@ -18,6 +18,7 @@ from common.djangoapps.student.tests.factories import UserFactory, UserProfileFactory from openedx.core.djangoapps.user_api.tests.test_views import UserAPITestCase from openedx.core.djangolib.testing.utils import skip_unless_lms +from common.test.utils import assert_dict_contains_subset @skip_unless_lms @@ -83,21 +84,22 @@ def test_send_registration_event(self): user = User.objects.get(username=self.user_info.get("username")) self.assertTrue(self.receiver_called) - self.assertDictContainsSubset( + assert_dict_contains_subset( + self, { "signal": STUDENT_REGISTRATION_COMPLETED, "sender": None, "user": UserData( - pii=UserPersonalData( - username=user.username, - email=user.email, - name=user.profile.name, - ), - id=user.id, - is_active=user.is_active, - ), + pii=UserPersonalData( + username=user.username, + email=user.email, + name=user.profile.name, + ), + id=user.id, + is_active=user.is_active, + ), }, - event_receiver.call_args.kwargs + event_receiver.call_args.kwargs, ) @@ -165,19 +167,20 @@ def test_send_login_event(self): user = User.objects.get(username=self.user.username) self.assertTrue(self.receiver_called) - self.assertDictContainsSubset( + assert_dict_contains_subset( + self, { "signal": SESSION_LOGIN_COMPLETED, "sender": None, "user": UserData( - pii=UserPersonalData( - username=user.username, - email=user.email, - name=user.profile.name, - ), - id=user.id, - is_active=user.is_active, - ), + pii=UserPersonalData( + username=user.username, + email=user.email, + name=user.profile.name, + ), + id=user.id, + is_active=user.is_active, + ), }, - event_receiver.call_args.kwargs + event_receiver.call_args.kwargs, ) diff --git a/openedx/core/djangoapps/user_authn/views/tests/test_login.py b/openedx/core/djangoapps/user_authn/views/tests/test_login.py index b00702ee25da..c8bfa082900b 100644 --- a/openedx/core/djangoapps/user_authn/views/tests/test_login.py +++ b/openedx/core/djangoapps/user_authn/views/tests/test_login.py @@ -44,6 +44,7 @@ from openedx.features.enterprise_support.tests.factories import EnterpriseCustomerUserFactory from common.djangoapps.student.models import LoginFailures from common.djangoapps.util.password_policy_validators import DEFAULT_MAX_PASSWORD_LENGTH +from common.test.utils import assert_dict_contains_subset @ddt.ddt @@ -544,7 +545,7 @@ def test_unicode_mktg_cookie_names(self): expected = { 'target': '/', } - self.assertDictContainsSubset(expected, response.context_data) + assert_dict_contains_subset(self, expected, response.context_data) @patch.dict("django.conf.settings.FEATURES", {'SQUELCH_PII_IN_LOGS': True}) def test_logout_logging_no_pii(self): diff --git a/openedx/core/djangoapps/user_authn/views/tests/test_logout.py b/openedx/core/djangoapps/user_authn/views/tests/test_logout.py index 77c21c86e1b1..a81d11c42cf4 100644 --- a/openedx/core/djangoapps/user_authn/views/tests/test_logout.py +++ b/openedx/core/djangoapps/user_authn/views/tests/test_logout.py @@ -14,6 +14,7 @@ from openedx.core.djangoapps.oauth_dispatch.tests.factories import ApplicationFactory from openedx.core.djangolib.testing.utils import skip_unless_lms from common.djangoapps.student.tests.factories import UserFactory +from common.test.utils import assert_dict_contains_subset @skip_unless_lms @@ -76,14 +77,14 @@ def test_logout_redirect_success(self, redirect_url, host): expected = { 'target': urllib.parse.unquote(redirect_url), } - self.assertDictContainsSubset(expected, response.context_data) + assert_dict_contains_subset(self, expected, response.context_data) def test_no_redirect_supplied(self): response = self.client.get(reverse('logout'), HTTP_HOST='testserver') expected = { 'target': '/', } - self.assertDictContainsSubset(expected, response.context_data) + assert_dict_contains_subset(self, expected, response.context_data) @ddt.data( ('https://www.amazon.org', 'edx.org'), @@ -100,7 +101,7 @@ def test_logout_redirect_failure(self, redirect_url, host): expected = { 'target': '/', } - self.assertDictContainsSubset(expected, response.context_data) + assert_dict_contains_subset(self, expected, response.context_data) def test_client_logout(self): """ Verify the context includes a list of the logout URIs of the authenticated OpenID Connect clients. @@ -113,7 +114,7 @@ def test_client_logout(self): 'logout_uris': [], 'target': '/', } - self.assertDictContainsSubset(expected, response.context_data) + assert_dict_contains_subset(self, expected, response.context_data) @mock.patch( 'django.conf.settings.IDA_LOGOUT_URI_LIST', @@ -138,7 +139,7 @@ def test_client_logout_with_dot_idas(self): 'logout_uris': expected_logout_uris, 'target': '/', } - self.assertDictContainsSubset(expected, response.context_data) + assert_dict_contains_subset(self, expected, response.context_data) @mock.patch( 'django.conf.settings.IDA_LOGOUT_URI_LIST', @@ -161,7 +162,7 @@ def test_client_logout_with_dot_idas_and_no_oidc_idas(self): 'logout_uris': expected_logout_uris, 'target': '/', } - self.assertDictContainsSubset(expected, response.context_data) + assert_dict_contains_subset(self, expected, response.context_data) def test_filter_referring_service(self): """ Verify that, if the user is directed to the logout page from a service, that service's logout URL @@ -174,7 +175,7 @@ def test_filter_referring_service(self): 'target': '/', 'show_tpa_logout_link': False, } - self.assertDictContainsSubset(expected, response.context_data) + assert_dict_contains_subset(self, expected, response.context_data) def test_learner_portal_logout_having_idp_logout_url(self): """ @@ -194,7 +195,7 @@ def test_learner_portal_logout_having_idp_logout_url(self): 'tpa_logout_url': idp_logout_url, 'show_tpa_logout_link': True, } - self.assertDictContainsSubset(expected, response.context_data) + assert_dict_contains_subset(self, expected, response.context_data) @mock.patch('django.conf.settings.TPA_AUTOMATIC_LOGOUT_ENABLED', True) def test_automatic_tpa_logout_url_redirect(self): @@ -214,7 +215,7 @@ def test_automatic_tpa_logout_url_redirect(self): expected = { 'target': idp_logout_url, } - self.assertDictContainsSubset(expected, response.context_data) + assert_dict_contains_subset(self, expected, response.context_data) @mock.patch('django.conf.settings.TPA_AUTOMATIC_LOGOUT_ENABLED', True) def test_no_automatic_tpa_logout_without_logout_url(self): @@ -241,4 +242,4 @@ def test_logout_redirect_failure_with_xss_vulnerability(self, redirect_url, host expected = { 'target': nh3.clean(urllib.parse.unquote(redirect_url)), } - self.assertDictContainsSubset(expected, response.context_data) + assert_dict_contains_subset(self, expected, response.context_data) diff --git a/openedx/features/enterprise_support/tests/test_logout.py b/openedx/features/enterprise_support/tests/test_logout.py index c83b67c3a5e6..cbec23397795 100644 --- a/openedx/features/enterprise_support/tests/test_logout.py +++ b/openedx/features/enterprise_support/tests/test_logout.py @@ -19,6 +19,7 @@ factories ) from openedx.features.enterprise_support.tests.mixins.enterprise import EnterpriseServiceMockMixin +from common.test.utils import assert_dict_contains_subset @ddt.ddt @@ -60,7 +61,7 @@ def test_logout_enterprise_target(self, redirect_url, enterprise_target): expected = { 'enterprise_target': enterprise_target, } - self.assertDictContainsSubset(expected, response.context_data) + assert_dict_contains_subset(self, expected, response.context_data) if enterprise_target: self.assertContains(response, 'We are signing you in.') diff --git a/xmodule/modulestore/tests/test_mixed_modulestore.py b/xmodule/modulestore/tests/test_mixed_modulestore.py index 1f4bcfac7b0c..a7fafaf8ad2a 100644 --- a/xmodule/modulestore/tests/test_mixed_modulestore.py +++ b/xmodule/modulestore/tests/test_mixed_modulestore.py @@ -63,6 +63,7 @@ from xmodule.modulestore.xml_importer import LocationMixin, import_course_from_xml from xmodule.tests import DATA_DIR, CourseComparisonTest from xmodule.x_module import XModuleMixin +from common.test.utils import assert_dict_contains_subset if not settings.configured: settings.configure() @@ -813,15 +814,16 @@ def test_course_create_event(self, default_ms): event_receiver.assert_called() - self.assertDictContainsSubset( + assert_dict_contains_subset( + self, { "signal": COURSE_CREATED, "sender": None, "course": CourseData( - course_key=test_course.id, - ), + course_key=test_course.id, + ), }, - event_receiver.call_args.kwargs + event_receiver.call_args.kwargs, ) @ddt.data(ModuleStoreEnum.Type.split) @@ -891,16 +893,17 @@ def test_xblock_publish_event(self, default_ms): self.store.publish(sequential.location, self.user_id) event_receiver.assert_called() - self.assertDictContainsSubset( + assert_dict_contains_subset( + self, { "signal": XBLOCK_PUBLISHED, "sender": None, "xblock_info": XBlockData( - usage_key=sequential.location, - block_type=sequential.location.block_type, - ), + usage_key=sequential.location, + block_type=sequential.location.block_type, + ), }, - event_receiver.call_args.kwargs + event_receiver.call_args.kwargs, ) @ddt.data(ModuleStoreEnum.Type.split) @@ -925,16 +928,17 @@ def test_xblock_delete_event(self, default_ms): self.store.delete_item(vertical.location, self.user_id) event_receiver.assert_called() - self.assertDictContainsSubset( + assert_dict_contains_subset( + self, { "signal": XBLOCK_DELETED, "sender": None, "xblock_info": XBlockData( - usage_key=vertical.location, - block_type=vertical.location.block_type, - ), + usage_key=vertical.location, + block_type=vertical.location.block_type, + ), }, - event_receiver.call_args.kwargs + event_receiver.call_args.kwargs, ) def setup_has_changes(self, default_ms): From ebaf5e64de3223b3ea00b9694abf4015c648f719 Mon Sep 17 00:00:00 2001 From: usamasadiq Date: Mon, 13 Oct 2025 19:00:45 +0500 Subject: [PATCH 013/351] fix: fix pylint warning --- .../tests/test_tasks_helper.py | 24 +++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) diff --git a/lms/djangoapps/instructor_task/tests/test_tasks_helper.py b/lms/djangoapps/instructor_task/tests/test_tasks_helper.py index 1f8e0b637d68..4144dd95680f 100644 --- a/lms/djangoapps/instructor_task/tests/test_tasks_helper.py +++ b/lms/djangoapps/instructor_task/tests/test_tasks_helper.py @@ -850,7 +850,11 @@ def test_no_problems(self, use_tempfile, _): """ with patch(USE_ON_DISK_GRADE_REPORT, return_value=use_tempfile): result = ProblemGradeReport.generate(None, None, self.course.id, {}, 'graded') - assert_dict_contains_subset(self, {'action_name': 'graded', 'attempted': 2, 'succeeded': 2, 'failed': 0}, result) + assert_dict_contains_subset( + self, + {'action_name': 'graded', 'attempted': 2, 'succeeded': 2, 'failed': 0}, + result + ) self.verify_rows_in_csv([ dict(list(zip( self.csv_header_row, @@ -876,7 +880,11 @@ def test_single_problem(self, use_tempfile, _): self.submit_student_answer(self.student_1.username, 'Problem1', ['Option 1']) with patch(USE_ON_DISK_GRADE_REPORT, return_value=use_tempfile): result = ProblemGradeReport.generate(None, None, self.course.id, {}, 'graded') - assert_dict_contains_subset(self, {'action_name': 'graded', 'attempted': 2, 'succeeded': 2, 'failed': 0}, result) + assert_dict_contains_subset( + self, + {'action_name': 'graded', 'attempted': 2, 'succeeded': 2, 'failed': 0}, + result + ) problem_name = 'Homework 1: Subsection - Problem1' header_row = self.csv_header_row + [problem_name + ' (Earned)', problem_name + ' (Possible)'] self.verify_rows_in_csv([ @@ -946,7 +954,11 @@ def test_inactive_enrollment_included(self, use_tempfile, _): self.submit_student_answer(self.student_1.username, 'Problem1', ['Option 1']) with patch(USE_ON_DISK_GRADE_REPORT, return_value=use_tempfile): result = ProblemGradeReport.generate(None, None, self.course.id, {}, 'graded') - assert_dict_contains_subset(self, {'action_name': 'graded', 'attempted': 3, 'succeeded': 3, 'failed': 0}, result) + assert_dict_contains_subset( + self, + {'action_name': 'graded', 'attempted': 3, 'succeeded': 3, 'failed': 0}, + result + ) problem_name = 'Homework 1: Subsection - Problem1' header_row = self.csv_header_row + [problem_name + ' (Earned)', problem_name + ' (Possible)'] self.verify_rows_in_csv([ @@ -1500,7 +1512,11 @@ def test_unicode_email_addresses(self): result = upload_may_enroll_csv(None, None, self.course.id, task_input, 'calculated') # This assertion simply confirms that the generation completed with no errors num_enrollments = len(enrollments) - assert_dict_contains_subset(self, {'attempted': num_enrollments, 'succeeded': num_enrollments, 'failed': 0}, result) + assert_dict_contains_subset( + self, + {'attempted': num_enrollments, 'succeeded': num_enrollments, 'failed': 0}, + result + ) class MockDefaultStorage: From 8aa2970c519cc5fc59a0f003f8b5967964d1a802 Mon Sep 17 00:00:00 2001 From: usamasadiq Date: Mon, 13 Oct 2025 22:53:37 +0500 Subject: [PATCH 014/351] fix: fix pycodestyle error --- .../contentstore/views/tests/test_block.py | 8 +- .../djangoapps/student/tests/test_events.py | 150 +++++++++--------- .../certificates/tests/test_events.py | 108 ++++++------- lms/djangoapps/grades/tests/test_events.py | 90 +++++------ .../course_groups/tests/test_events.py | 28 ++-- .../user_authn/views/tests/test_events.py | 32 ++-- .../tests/test_mixed_modulestore.py | 16 +- 7 files changed, 216 insertions(+), 216 deletions(-) diff --git a/cms/djangoapps/contentstore/views/tests/test_block.py b/cms/djangoapps/contentstore/views/tests/test_block.py index e01263259bcd..2b21a9b9b970 100644 --- a/cms/djangoapps/contentstore/views/tests/test_block.py +++ b/cms/djangoapps/contentstore/views/tests/test_block.py @@ -870,10 +870,10 @@ def test_duplicate_event(self): "signal": XBLOCK_DUPLICATED, "sender": None, "xblock_info": DuplicatedXBlockData( - usage_key=usage_key, - block_type=usage_key.block_type, - source_usage_key=self.vert_usage_key, - ), + usage_key=usage_key, + block_type=usage_key.block_type, + source_usage_key=self.vert_usage_key, + ), }, event_receiver.call_args.kwargs, ) diff --git a/common/djangoapps/student/tests/test_events.py b/common/djangoapps/student/tests/test_events.py index 9d57c8ca8567..7c9d625118d0 100644 --- a/common/djangoapps/student/tests/test_events.py +++ b/common/djangoapps/student/tests/test_events.py @@ -278,23 +278,23 @@ def test_enrollment_created_event_emitted(self): "signal": COURSE_ENROLLMENT_CREATED, "sender": None, "enrollment": CourseEnrollmentData( - user=UserData( - pii=UserPersonalData( - username=self.user.username, - email=self.user.email, - name=self.user.profile.name, - ), - id=self.user.id, - is_active=self.user.is_active, - ), - course=CourseData( - course_key=self.course.id, - display_name=self.course.display_name, - ), - mode=enrollment.mode, - is_active=enrollment.is_active, - creation_date=enrollment.created, - ), + user=UserData( + pii=UserPersonalData( + username=self.user.username, + email=self.user.email, + name=self.user.profile.name, + ), + id=self.user.id, + is_active=self.user.is_active, + ), + course=CourseData( + course_key=self.course.id, + display_name=self.course.display_name, + ), + mode=enrollment.mode, + is_active=enrollment.is_active, + creation_date=enrollment.created, + ), }, event_receiver.call_args.kwargs, ) @@ -322,23 +322,23 @@ def test_enrollment_changed_event_emitted(self): "signal": COURSE_ENROLLMENT_CHANGED, "sender": None, "enrollment": CourseEnrollmentData( - user=UserData( - pii=UserPersonalData( - username=self.user.username, - email=self.user.email, - name=self.user.profile.name, - ), - id=self.user.id, - is_active=self.user.is_active, - ), - course=CourseData( - course_key=self.course.id, - display_name=self.course.display_name, - ), - mode=enrollment.mode, - is_active=enrollment.is_active, - creation_date=enrollment.created, - ), + user=UserData( + pii=UserPersonalData( + username=self.user.username, + email=self.user.email, + name=self.user.profile.name, + ), + id=self.user.id, + is_active=self.user.is_active, + ), + course=CourseData( + course_key=self.course.id, + display_name=self.course.display_name, + ), + mode=enrollment.mode, + is_active=enrollment.is_active, + creation_date=enrollment.created, + ), }, event_receiver.call_args.kwargs, ) @@ -366,23 +366,23 @@ def test_unenrollment_completed_event_emitted(self): "signal": COURSE_UNENROLLMENT_COMPLETED, "sender": None, "enrollment": CourseEnrollmentData( - user=UserData( - pii=UserPersonalData( - username=self.user.username, - email=self.user.email, - name=self.user.profile.name, - ), - id=self.user.id, - is_active=self.user.is_active, - ), - course=CourseData( - course_key=self.course.id, - display_name=self.course.display_name, - ), - mode=enrollment.mode, - is_active=False, - creation_date=enrollment.created, - ), + user=UserData( + pii=UserPersonalData( + username=self.user.username, + email=self.user.email, + name=self.user.profile.name, + ), + id=self.user.id, + is_active=self.user.is_active, + ), + course=CourseData( + course_key=self.course.id, + display_name=self.course.display_name, + ), + mode=enrollment.mode, + is_active=False, + creation_date=enrollment.created, + ), }, event_receiver.call_args.kwargs, ) @@ -440,18 +440,18 @@ def test_access_role_created_event_emitted(self, AccessRole): "signal": COURSE_ACCESS_ROLE_ADDED, "sender": None, "course_access_role_data": CourseAccessRoleData( - user=UserData( - pii=UserPersonalData( - username=self.user.username, - email=self.user.email, - ), - id=self.user.id, - is_active=self.user.is_active, - ), - course_key=self.course_key, - org_key=self.course_key.org, - role=role._role_name, # pylint: disable=protected-access - ), + user=UserData( + pii=UserPersonalData( + username=self.user.username, + email=self.user.email, + ), + id=self.user.id, + is_active=self.user.is_active, + ), + course_key=self.course_key, + org_key=self.course_key.org, + role=role._role_name, # pylint: disable=protected-access + ), }, event_receiver.call_args.kwargs, ) @@ -479,18 +479,18 @@ def test_access_role_removed_event_emitted(self, AccessRole): "signal": COURSE_ACCESS_ROLE_REMOVED, "sender": None, "course_access_role_data": CourseAccessRoleData( - user=UserData( - pii=UserPersonalData( - username=self.user.username, - email=self.user.email, - ), - id=self.user.id, - is_active=self.user.is_active, - ), - course_key=self.course_key, - org_key=self.course_key.org, - role=role._role_name, # pylint: disable=protected-access - ), + user=UserData( + pii=UserPersonalData( + username=self.user.username, + email=self.user.email, + ), + id=self.user.id, + is_active=self.user.is_active, + ), + course_key=self.course_key, + org_key=self.course_key.org, + role=role._role_name, # pylint: disable=protected-access + ), }, event_receiver.call_args.kwargs, ) diff --git a/lms/djangoapps/certificates/tests/test_events.py b/lms/djangoapps/certificates/tests/test_events.py index fb890c53dd6d..eb1f215e4953 100644 --- a/lms/djangoapps/certificates/tests/test_events.py +++ b/lms/djangoapps/certificates/tests/test_events.py @@ -101,24 +101,24 @@ def test_send_certificate_created_event(self): "signal": CERTIFICATE_CREATED, "sender": None, "certificate": CertificateData( - user=UserData( - pii=UserPersonalData( - username=certificate.user.username, - email=certificate.user.email, - name=certificate.user.profile.name, - ), - id=certificate.user.id, - is_active=certificate.user.is_active, - ), - course=CourseData( - course_key=certificate.course_id, - ), - mode=certificate.mode, - grade=certificate.grade, - current_status=certificate.status, - download_url=certificate.download_url, - name=certificate.name, - ), + user=UserData( + pii=UserPersonalData( + username=certificate.user.username, + email=certificate.user.email, + name=certificate.user.profile.name, + ), + id=certificate.user.id, + is_active=certificate.user.is_active, + ), + course=CourseData( + course_key=certificate.course_id, + ), + mode=certificate.mode, + grade=certificate.grade, + current_status=certificate.status, + download_url=certificate.download_url, + name=certificate.name, + ), }, event_receiver.call_args.kwargs, ) @@ -155,24 +155,24 @@ def test_send_certificate_changed_event(self): "signal": CERTIFICATE_CHANGED, "sender": None, "certificate": CertificateData( - user=UserData( - pii=UserPersonalData( - username=certificate.user.username, - email=certificate.user.email, - name=certificate.user.profile.name, - ), - id=certificate.user.id, - is_active=certificate.user.is_active, - ), - course=CourseData( - course_key=certificate.course_id, - ), - mode=certificate.mode, - grade=certificate.grade, - current_status=certificate.status, - download_url=certificate.download_url, - name=certificate.name, - ), + user=UserData( + pii=UserPersonalData( + username=certificate.user.username, + email=certificate.user.email, + name=certificate.user.profile.name, + ), + id=certificate.user.id, + is_active=certificate.user.is_active, + ), + course=CourseData( + course_key=certificate.course_id, + ), + mode=certificate.mode, + grade=certificate.grade, + current_status=certificate.status, + download_url=certificate.download_url, + name=certificate.name, + ), }, event_receiver.call_args.kwargs, ) @@ -208,24 +208,24 @@ def test_send_certificate_revoked_event(self): "signal": CERTIFICATE_REVOKED, "sender": None, "certificate": CertificateData( - user=UserData( - pii=UserPersonalData( - username=certificate.user.username, - email=certificate.user.email, - name=certificate.user.profile.name, - ), - id=certificate.user.id, - is_active=certificate.user.is_active, - ), - course=CourseData( - course_key=certificate.course_id, - ), - mode=certificate.mode, - grade=certificate.grade, - current_status=certificate.status, - download_url=certificate.download_url, - name=certificate.name, - ), + user=UserData( + pii=UserPersonalData( + username=certificate.user.username, + email=certificate.user.email, + name=certificate.user.profile.name, + ), + id=certificate.user.id, + is_active=certificate.user.is_active, + ), + course=CourseData( + course_key=certificate.course_id, + ), + mode=certificate.mode, + grade=certificate.grade, + current_status=certificate.status, + download_url=certificate.download_url, + name=certificate.name, + ), }, event_receiver.call_args.kwargs, ) diff --git a/lms/djangoapps/grades/tests/test_events.py b/lms/djangoapps/grades/tests/test_events.py index 93c440bdb536..4b3063265aff 100644 --- a/lms/djangoapps/grades/tests/test_events.py +++ b/lms/djangoapps/grades/tests/test_events.py @@ -97,17 +97,17 @@ def test_persistent_grade_event_emitted(self): "signal": PERSISTENT_GRADE_SUMMARY_CHANGED, "sender": None, "grade": PersistentCourseGradeData( - user_id=self.params["user_id"], - course=CourseData( - course_key=self.params["course_id"], - ), - course_edited_timestamp=self.params["course_edited_timestamp"], - course_version=self.params["course_version"], - grading_policy_hash='', - percent_grade=self.params["percent_grade"], - letter_grade=self.params["letter_grade"], - passed_timestamp=grade.passed_timestamp - ) + user_id=self.params["user_id"], + course=CourseData( + course_key=self.params["course_id"], + ), + course_edited_timestamp=self.params["course_edited_timestamp"], + course_version=self.params["course_version"], + grading_policy_hash='', + percent_grade=self.params["percent_grade"], + letter_grade=self.params["letter_grade"], + passed_timestamp=grade.passed_timestamp + ) }, event_receiver.call_args.kwargs, ) @@ -159,20 +159,20 @@ def test_course_passing_status_updated_emitted(self): "signal": COURSE_PASSING_STATUS_UPDATED, "sender": None, "course_passing_status": CoursePassingStatusData( - is_passing=True, - user=UserData( - pii=UserPersonalData( - username=self.user.username, - email=self.user.email, - name=self.user.get_full_name(), - ), - id=self.user.id, - is_active=self.user.is_active, - ), - course=CourseData( - course_key=self.course.id, - ), - ), + is_passing=True, + user=UserData( + pii=UserPersonalData( + username=self.user.username, + email=self.user.email, + name=self.user.get_full_name(), + ), + id=self.user.id, + is_active=self.user.is_active, + ), + course=CourseData( + course_key=self.course.id, + ), + ), }, event_receiver.call_args.kwargs, ) @@ -233,26 +233,26 @@ def test_ccx_course_passing_status_updated_emitted(self): "signal": CCX_COURSE_PASSING_STATUS_UPDATED, "sender": None, "course_passing_status": CcxCoursePassingStatusData( - is_passing=True, - user=UserData( - pii=UserPersonalData( - username=self.user.username, - email=self.user.email, - name=self.user.get_full_name(), - ), - id=self.user.id, - is_active=self.user.is_active, - ), - course=CcxCourseData( - ccx_course_key=self.ccx_locator, - master_course_key=self.course.id, - display_name="", - coach_email="", - start=None, - end=None, - max_students_allowed=self.ccx.max_student_enrollments_allowed, - ), - ), + is_passing=True, + user=UserData( + pii=UserPersonalData( + username=self.user.username, + email=self.user.email, + name=self.user.get_full_name(), + ), + id=self.user.id, + is_active=self.user.is_active, + ), + course=CcxCourseData( + ccx_course_key=self.ccx_locator, + master_course_key=self.course.id, + display_name="", + coach_email="", + start=None, + end=None, + max_students_allowed=self.ccx.max_student_enrollments_allowed, + ), + ), }, event_receiver.call_args.kwargs, ) diff --git a/openedx/core/djangoapps/course_groups/tests/test_events.py b/openedx/core/djangoapps/course_groups/tests/test_events.py index 40a04dea8c8b..f11e95088fd0 100644 --- a/openedx/core/djangoapps/course_groups/tests/test_events.py +++ b/openedx/core/djangoapps/course_groups/tests/test_events.py @@ -97,20 +97,20 @@ def test_send_cohort_membership_changed_event(self): "signal": COHORT_MEMBERSHIP_CHANGED, "sender": None, "cohort": CohortData( - user=UserData( - pii=UserPersonalData( - username=cohort_membership.user.username, - email=cohort_membership.user.email, - name=cohort_membership.user.profile.name, - ), - id=cohort_membership.user.id, - is_active=cohort_membership.user.is_active, - ), - course=CourseData( - course_key=cohort_membership.course_id, - ), - name=cohort_membership.course_user_group.name, - ), + user=UserData( + pii=UserPersonalData( + username=cohort_membership.user.username, + email=cohort_membership.user.email, + name=cohort_membership.user.profile.name, + ), + id=cohort_membership.user.id, + is_active=cohort_membership.user.is_active, + ), + course=CourseData( + course_key=cohort_membership.course_id, + ), + name=cohort_membership.course_user_group.name, + ), }, event_receiver.call_args.kwargs, ) diff --git a/openedx/core/djangoapps/user_authn/views/tests/test_events.py b/openedx/core/djangoapps/user_authn/views/tests/test_events.py index 4460e8c627b4..7efd4e4cf5c0 100644 --- a/openedx/core/djangoapps/user_authn/views/tests/test_events.py +++ b/openedx/core/djangoapps/user_authn/views/tests/test_events.py @@ -90,14 +90,14 @@ def test_send_registration_event(self): "signal": STUDENT_REGISTRATION_COMPLETED, "sender": None, "user": UserData( - pii=UserPersonalData( - username=user.username, - email=user.email, - name=user.profile.name, - ), - id=user.id, - is_active=user.is_active, - ), + pii=UserPersonalData( + username=user.username, + email=user.email, + name=user.profile.name, + ), + id=user.id, + is_active=user.is_active, + ), }, event_receiver.call_args.kwargs, ) @@ -173,14 +173,14 @@ def test_send_login_event(self): "signal": SESSION_LOGIN_COMPLETED, "sender": None, "user": UserData( - pii=UserPersonalData( - username=user.username, - email=user.email, - name=user.profile.name, - ), - id=user.id, - is_active=user.is_active, - ), + pii=UserPersonalData( + username=user.username, + email=user.email, + name=user.profile.name, + ), + id=user.id, + is_active=user.is_active, + ), }, event_receiver.call_args.kwargs, ) diff --git a/xmodule/modulestore/tests/test_mixed_modulestore.py b/xmodule/modulestore/tests/test_mixed_modulestore.py index a7fafaf8ad2a..f82d1e57c570 100644 --- a/xmodule/modulestore/tests/test_mixed_modulestore.py +++ b/xmodule/modulestore/tests/test_mixed_modulestore.py @@ -820,8 +820,8 @@ def test_course_create_event(self, default_ms): "signal": COURSE_CREATED, "sender": None, "course": CourseData( - course_key=test_course.id, - ), + course_key=test_course.id, + ), }, event_receiver.call_args.kwargs, ) @@ -899,9 +899,9 @@ def test_xblock_publish_event(self, default_ms): "signal": XBLOCK_PUBLISHED, "sender": None, "xblock_info": XBlockData( - usage_key=sequential.location, - block_type=sequential.location.block_type, - ), + usage_key=sequential.location, + block_type=sequential.location.block_type, + ), }, event_receiver.call_args.kwargs, ) @@ -934,9 +934,9 @@ def test_xblock_delete_event(self, default_ms): "signal": XBLOCK_DELETED, "sender": None, "xblock_info": XBlockData( - usage_key=vertical.location, - block_type=vertical.location.block_type, - ), + usage_key=vertical.location, + block_type=vertical.location.block_type, + ), }, event_receiver.call_args.kwargs, ) From 3db4399f7451f9fccca26494f873d5550dc9afb1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Chris=20Ch=C3=A1vez?= Date: Mon, 13 Oct 2025 16:34:36 -0500 Subject: [PATCH 015/351] feat: bulk modulestore migration [FC-0097] (#37381) - Adds the task, python api, and rest api view for bulk migration. - Refactor the code to share code between single migration and bulk migration. --- cms/djangoapps/modulestore_migrator/api.py | 48 +- ..._alter_modulestoremigration_task_status.py | 20 + cms/djangoapps/modulestore_migrator/models.py | 8 +- .../rest_api/v1/serializers.py | 77 +- .../modulestore_migrator/rest_api/v1/urls.py | 5 +- .../modulestore_migrator/rest_api/v1/views.py | 198 ++++- cms/djangoapps/modulestore_migrator/tasks.py | 716 +++++++++++++++--- .../modulestore_migrator/tests/test_api.py | 39 + .../modulestore_migrator/tests/test_tasks.py | 527 ++++++++++++- 9 files changed, 1468 insertions(+), 170 deletions(-) create mode 100644 cms/djangoapps/modulestore_migrator/migrations/0002_alter_modulestoremigration_task_status.py diff --git a/cms/djangoapps/modulestore_migrator/api.py b/cms/djangoapps/modulestore_migrator/api.py index 25c26b007923..d084be764a2e 100644 --- a/cms/djangoapps/modulestore_migrator/api.py +++ b/cms/djangoapps/modulestore_migrator/api.py @@ -15,6 +15,7 @@ __all__ = ( "start_migration_to_library", + "start_bulk_migration_to_library", "is_successfully_migrated", "get_migration_info", ) @@ -46,7 +47,6 @@ def start_migration_to_library( return tasks.migrate_from_modulestore.delay( user_id=user.id, source_pk=source.id, - target_package_pk=target_package_id, target_library_key=str(target_library_key), target_collection_pk=target_collection_id, composition_level=composition_level, @@ -56,6 +56,52 @@ def start_migration_to_library( ) +def start_bulk_migration_to_library( + *, + user: AuthUser, + source_key_list: list[LearningContextKey], + target_library_key: LibraryLocatorV2, + target_collection_slug_list: list[str | None] | None = None, + create_collections: bool = False, + composition_level: str, + repeat_handling_strategy: str, + preserve_url_slugs: bool, + forward_source_to_target: bool, +) -> AsyncResult: + """ + Import a list of courses or legacy libraries into a V2 library (or, a collections within a V2 library). + """ + target_library = get_library(target_library_key) + # get_library ensures that the library is connected to a learning package. + target_package_id: int = target_library.learning_package_id # type: ignore[assignment] + + sources_pks: list[int] = [] + for source_key in source_key_list: + source, _ = ModulestoreSource.objects.get_or_create(key=source_key) + sources_pks.append(source.id) + + target_collection_pks: list[int | None] = [] + if target_collection_slug_list: + for target_collection_slug in target_collection_slug_list: + if target_collection_slug: + target_collection_id = get_collection(target_package_id, target_collection_slug).id + target_collection_pks.append(target_collection_id) + else: + target_collection_pks.append(None) + + return tasks.bulk_migrate_from_modulestore.delay( + user_id=user.id, + sources_pks=sources_pks, + target_library_key=str(target_library_key), + target_collection_pks=target_collection_pks, + create_collections=create_collections, + composition_level=composition_level, + repeat_handling_strategy=repeat_handling_strategy, + preserve_url_slugs=preserve_url_slugs, + forward_source_to_target=forward_source_to_target, + ) + + def is_successfully_migrated(source_key: CourseKey | LibraryLocator) -> bool: """ Check if the source course/library has been migrated successfully. diff --git a/cms/djangoapps/modulestore_migrator/migrations/0002_alter_modulestoremigration_task_status.py b/cms/djangoapps/modulestore_migrator/migrations/0002_alter_modulestoremigration_task_status.py new file mode 100644 index 000000000000..ed3a299c30c0 --- /dev/null +++ b/cms/djangoapps/modulestore_migrator/migrations/0002_alter_modulestoremigration_task_status.py @@ -0,0 +1,20 @@ +# Generated by Django 4.2.24 on 2025-09-29 20:28 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('user_tasks', '0004_url_textfield'), + ('modulestore_migrator', '0001_initial'), + ] + + operations = [ + migrations.AlterField( + model_name='modulestoremigration', + name='task_status', + field=models.ForeignKey(help_text='Tracks the status of the task which is executing this migration. In a bulk migration, the same task can be multiple migrations', on_delete=django.db.models.deletion.RESTRICT, related_name='migrations', to='user_tasks.usertaskstatus'), + ), + ] diff --git a/cms/djangoapps/modulestore_migrator/models.py b/cms/djangoapps/modulestore_migrator/models.py index 05bea5bfe1e9..cf14a3483519 100644 --- a/cms/djangoapps/modulestore_migrator/models.py +++ b/cms/djangoapps/modulestore_migrator/models.py @@ -107,10 +107,14 @@ class ModulestoreMigration(models.Model): ) ## MIGRATION ARTIFACTS - task_status = models.OneToOneField( + task_status = models.ForeignKey( UserTaskStatus, on_delete=models.RESTRICT, - help_text=_("Tracks the status of the task which is executing this migration"), + help_text=_( + "Tracks the status of the task which is executing this migration. " + "In a bulk migration, the same task can be multiple migrations" + ), + related_name="migrations", ) change_log = models.ForeignKey( DraftChangeLog, diff --git a/cms/djangoapps/modulestore_migrator/rest_api/v1/serializers.py b/cms/djangoapps/modulestore_migrator/rest_api/v1/serializers.py index df981e896176..b9573e16ab98 100644 --- a/cms/djangoapps/modulestore_migrator/rest_api/v1/serializers.py +++ b/cms/djangoapps/modulestore_migrator/rest_api/v1/serializers.py @@ -12,9 +12,9 @@ from cms.djangoapps.modulestore_migrator.models import ModulestoreMigration -class ModulestoreMigrationSerializer(serializers.ModelSerializer): +class ModulestoreMigrationSerializer(serializers.Serializer): """ - Serializer for the course to library import creation API. + Serializer for the course or legacylibrary to library V2 import creation API. """ source = serializers.CharField( # type: ignore[assignment] @@ -22,7 +22,7 @@ class ModulestoreMigrationSerializer(serializers.ModelSerializer): required=True, ) target = serializers.CharField( - help_text="The target library key to import into.", + help_text="The target library V2 key to import into.", required=True, ) composition_level = serializers.ChoiceField( @@ -54,18 +54,6 @@ class ModulestoreMigrationSerializer(serializers.ModelSerializer): default=False, ) - class Meta: - model = ModulestoreMigration - fields = [ - 'source', - 'target', - 'target_collection_slug', - 'composition_level', - 'repeat_handling_strategy', - 'preserve_url_slugs', - 'forward_source_to_target', - ] - def get_fields(self): fields = super().get_fields() request = self.context.get('request') @@ -100,19 +88,74 @@ def get_forward_source_to_target(self, obj: ModulestoreMigration): def to_representation(self, instance): """ - Override to customize the serialized representation.""" + Override to customize the serialized representation. + """ data = super().to_representation(instance) # Custom logic for forward_source_to_target during serialization data['forward_source_to_target'] = self.get_forward_source_to_target(instance) return data +class BulkModulestoreMigrationSerializer(ModulestoreMigrationSerializer): + """ + Serializer for a bulk migration (of several courses or legacy libraries) to a V2 library. + """ + sources = serializers.ListField( + child=serializers.CharField(), + help_text="The list of sources course or legacy library keys to import from.", + required=True, + ) + + target_collection_slug_list = serializers.ListField( + child=serializers.CharField(), + help_text="The list of target collection slugs within the library to import into. Optional.", + required=False, + allow_empty=True, + default=None, + ) + + create_collections = serializers.BooleanField( + help_text=( + "If true and `target_collection_slug_list` is not set, " + "create the collections in the library where the import will be made" + ), + required=False, + default=False, + ) + + def get_fields(self): + fields = super().get_fields() + fields.pop("source", None) + fields.pop("target_collection_slug", None) + return fields + + def validate_sources(self, value): + """ + Validate all the source key format + """ + validated_sources = [] + for v in value: + try: + validated_sources.append(LearningContextKey.from_string(v)) + except InvalidKeyError as exc: + raise serializers.ValidationError(f"Invalid source key: {str(exc)}") from exc + return validated_sources + + def to_representation(self, instance): + """ + Override to customize the serialized representation. + """ + if isinstance(instance, list): + return [super().to_representation(obj) for obj in instance] + return super().to_representation(instance) + + class StatusWithModulestoreMigrationSerializer(StatusSerializer): """ Serializer for the import task status. """ - parameters = ModulestoreMigrationSerializer(source='modulestoremigration') + parameters = ModulestoreMigrationSerializer(source='migrations', many=True) class Meta: model = StatusSerializer.Meta.model diff --git a/cms/djangoapps/modulestore_migrator/rest_api/v1/urls.py b/cms/djangoapps/modulestore_migrator/rest_api/v1/urls.py index 22d509f16910..7f66dc5f6dd6 100644 --- a/cms/djangoapps/modulestore_migrator/rest_api/v1/urls.py +++ b/cms/djangoapps/modulestore_migrator/rest_api/v1/urls.py @@ -3,9 +3,10 @@ """ from rest_framework.routers import SimpleRouter -from .views import MigrationViewSet +from .views import MigrationViewSet, BulkMigrationViewSet ROUTER = SimpleRouter() -ROUTER.register(r'migrations', MigrationViewSet) +ROUTER.register(r'migrations', MigrationViewSet, basename='migrations') +ROUTER.register(r'bulk_migration', BulkMigrationViewSet, basename='bulk-migration') urlpatterns = ROUTER.urls diff --git a/cms/djangoapps/modulestore_migrator/rest_api/v1/views.py b/cms/djangoapps/modulestore_migrator/rest_api/v1/views.py index feeafb1b40d7..b7bdc0b0d6ef 100644 --- a/cms/djangoapps/modulestore_migrator/rest_api/v1/views.py +++ b/cms/djangoapps/modulestore_migrator/rest_api/v1/views.py @@ -3,6 +3,7 @@ """ import logging +import edx_api_doc_tools as apidocs from edx_rest_framework_extensions.auth.jwt.authentication import JwtAuthentication from edx_rest_framework_extensions.auth.session.authentication import SessionAuthenticationAllowInactiveUser from rest_framework.permissions import IsAdminUser @@ -11,10 +12,14 @@ from user_tasks.models import UserTaskStatus from user_tasks.views import StatusViewSet -from cms.djangoapps.modulestore_migrator.api import start_migration_to_library +from cms.djangoapps.modulestore_migrator.api import start_migration_to_library, start_bulk_migration_to_library from openedx.core.lib.api.authentication import BearerAuthenticationAllowInactiveUser -from .serializers import ModulestoreMigrationSerializer, StatusWithModulestoreMigrationSerializer +from .serializers import ( + StatusWithModulestoreMigrationSerializer, + ModulestoreMigrationSerializer, + BulkModulestoreMigrationSerializer, +) log = logging.getLogger(__name__) @@ -22,7 +27,7 @@ class MigrationViewSet(StatusViewSet): """ - Import course content from modulestore into a content library. + Import course content or legacy library content from modulestore into a content library. This viewset handles the import process, including creating the import task and retrieving the status of the import task. Meant to be used by admin users only. @@ -84,12 +89,14 @@ class MigrationViewSet(StatusViewSet): "modified": "2025-05-14T22:24:59.128068Z", "artifacts": [], "uuid": "3de23e5d-fd34-4a6f-bf02-b183374120f0", - "parameters": { - "source": "course-v1:OpenedX+DemoX+DemoCourse2", - "composition_level": "component", - "repeat_handling_strategy": "skip", - "preserve_url_slugs": false - } + "parameters": [ + { + "source": "course-v1:OpenedX+DemoX+DemoCourse2", + "composition_level": "component", + "repeat_handling_strategy": "skip", + "preserve_url_slugs": false + } + ] } """ @@ -103,33 +110,172 @@ class MigrationViewSet(StatusViewSet): def get_queryset(self): """ - Override the default queryset to filter by the import event and user. + Override the default queryset to filter by the migration event and user. """ - return StatusViewSet.queryset.filter(modulestoremigration__isnull=False, user=self.request.user) + return StatusViewSet.queryset.filter(migrations__isnull=False, user=self.request.user).distinct() + @apidocs.schema( + body=ModulestoreMigrationSerializer, + responses={ + 201: StatusWithModulestoreMigrationSerializer, + 401: "The requester is not authenticated.", + }, + summary="Start a modulestore to content library migration", + description=( + "Create a migration task to import course or legacy library content into " + "a content library.\n\n" + "**Request example**:\n\n" + "```json\n" + "{\n" + ' "source": "course-v1:edX+DemoX+2014_T1",\n' + ' "target": "library-v1:org1+lib_1",\n' + ' "composition_level": "unit",\n' + ' "repeat_handling_strategy": "update",\n' + ' "preserve_url_slugs": true\n' + "}\n" + "```" + ), + ) def create(self, request, *args, **kwargs): """ - Handle the import task creation. + Handle the migration task creation. """ serializer_data = ModulestoreMigrationSerializer(data=request.data) serializer_data.is_valid(raise_exception=True) validated_data = serializer_data.validated_data - try: - task = start_migration_to_library( - user=request.user, - source_key=validated_data['source'], - target_library_key=validated_data['target'], - target_collection_slug=validated_data['target_collection_slug'], - composition_level=validated_data['composition_level'], - repeat_handling_strategy=validated_data['repeat_handling_strategy'], - preserve_url_slugs=validated_data['preserve_url_slugs'], - forward_source_to_target=validated_data['forward_source_to_target'], - ) - except NotImplementedError as e: - log.exception(str(e)) - return Response({'error': str(e)}, status=status.HTTP_400_BAD_REQUEST) + task = start_migration_to_library( + user=request.user, + source_key=validated_data['source'], + target_library_key=validated_data['target'], + target_collection_slug=validated_data['target_collection_slug'], + composition_level=validated_data['composition_level'], + repeat_handling_strategy=validated_data['repeat_handling_strategy'], + preserve_url_slugs=validated_data['preserve_url_slugs'], + forward_source_to_target=validated_data['forward_source_to_target'], + ) + + task_status = UserTaskStatus.objects.get(task_id=task.id) + serializer = self.get_serializer(task_status) + + return Response(serializer.data, status=status.HTTP_201_CREATED) + + +class BulkMigrationViewSet(StatusViewSet): + """ + Import content of a list of courses or legacy libraries from modulestore into a content library. + + This viewset handles the import process, including creating the import task and + retrieving the status of the import task. Meant to be used by admin users only. + + API Endpoints + ------------ + POST /api/modulestore_migrator/v1/bulk-migration/ + Start the bulk import process. + + Request body: + { + "sources": ["", ""], + "target": "", + "composition_level": "", # Optional, defaults to "component" + "target_collection_slugs": ["", ""], # Optional + "create_collections": "" # Optional, defaults to false + "repeat_handling_strategy": "" # Optional, defaults to Skip + "preserve_url_slugs": "" # Optional, defaults to true + } + + Example request: + { + "sources": ["course-v1:edX+DemoX+2014_T1", "course-v1:edX+DemoX+2014_T2"], + "target": "library-v1:org1+lib_1", + "composition_level": "unit", + "repeat_handling_strategy": "update", + "preserve_url_slugs": true, + "create_collections": true + } + + Example response: + { + "state": "Succeeded", + "state_text": "Succeeded", # Translation into the current language of the current state + "completed_steps": 11, + "total_steps": 11, + "attempts": 1, + "created": "2025-05-14T22:24:37.048539Z", + "modified": "2025-05-14T22:24:59.128068Z", + "artifacts": [], + "uuid": "3de23e5d-fd34-4a6f-bf02-b183374120f0", + "parameters": [ + { + "source": "course-v1:edX+DemoX+2014_T1", + "composition_level": "unit", + "repeat_handling_strategy": "update", + "preserve_url_slugs": true + }, + { + "source": "course-v1:edX+DemoX+2014_T2", + "composition_level": "unit", + "repeat_handling_strategy": "update", + "preserve_url_slugs": true + }, + ] + } + + GET Not Alowed + """ + + permission_classes = (IsAdminUser,) + authentication_classes = ( + BearerAuthenticationAllowInactiveUser, + JwtAuthentication, + SessionAuthenticationAllowInactiveUser, + ) + serializer_class = StatusWithModulestoreMigrationSerializer + http_method_names = ["post"] + + @apidocs.schema( + body=BulkModulestoreMigrationSerializer, + responses={ + 201: StatusWithModulestoreMigrationSerializer, + 401: "The requester is not authenticated.", + }, + summary="Start a bulk modulestore to content library migration", + description=( + "Create a migration task to import multiple courses or legacy libraries " + "into a single content library.\n\n" + "**Request example**:\n\n" + "```json\n" + "{\n" + ' "sources": ["course-v1:edX+DemoX+2014_T1", "course-v1:edX+DemoX+2014_T2"],\n' + ' "target": "library-v1:org1+lib_1",\n' + ' "composition_level": "unit",\n' + ' "repeat_handling_strategy": "update",\n' + ' "preserve_url_slugs": true,\n' + ' "create_collections": true\n' + "}\n" + "```" + ), + ) + def create(self, request, *args, **kwargs): + """ + Handle the bulk migration task creation. + """ + serializer_data = BulkModulestoreMigrationSerializer(data=request.data) + serializer_data.is_valid(raise_exception=True) + validated_data = serializer_data.validated_data + + task = start_bulk_migration_to_library( + user=request.user, + source_key_list=validated_data['sources'], + target_library_key=validated_data['target'], + target_collection_slug_list=validated_data['target_collection_slug_list'], + create_collections=validated_data['create_collections'], + composition_level=validated_data['composition_level'], + repeat_handling_strategy=validated_data['repeat_handling_strategy'], + preserve_url_slugs=validated_data['preserve_url_slugs'], + forward_source_to_target=validated_data['forward_source_to_target'], + ) task_status = UserTaskStatus.objects.get(task_id=task.id) serializer = self.get_serializer(task_status) diff --git a/cms/djangoapps/modulestore_migrator/tasks.py b/cms/djangoapps/modulestore_migrator/tasks.py index 0c8520988738..75d477631f85 100644 --- a/cms/djangoapps/modulestore_migrator/tasks.py +++ b/cms/djangoapps/modulestore_migrator/tasks.py @@ -15,6 +15,7 @@ from celery.utils.log import get_task_logger from django.core.exceptions import ObjectDoesNotExist from django.utils.text import slugify +from django.db import transaction from edx_django_utils.monitoring import set_code_owner_attribute_from_module from lxml import etree from lxml.etree import _ElementTree as XmlTree @@ -37,8 +38,11 @@ PublishableEntityVersion ) from user_tasks.tasks import UserTask, UserTaskStatus +from xblock.core import XBlock +from django.utils.translation import gettext_lazy as _ from common.djangoapps.split_modulestore_django.models import SplitModulestoreCourseIndex +from common.djangoapps.util.date_utils import strftime_localized, DEFAULT_DATE_TIME_FORMAT from openedx.core.djangoapps.content_libraries import api as libraries_api from openedx.core.djangoapps.content_libraries.api import ContainerType, get_library from openedx.core.djangoapps.content_staging import api as staging_api @@ -71,6 +75,7 @@ class MigrationStep(Enum): MAPPING_OLD_TO_NEW = 'Saving map of legacy content to migrated content' FORWARDING = 'Forwarding legacy content to migrated content' POPULATING_COLLECTION = 'Assigning imported items to the specified collection' + BULK_MIGRATION_PREFIX = 'Migrating legacy content' class _MigrationTask(UserTask): @@ -83,7 +88,36 @@ def calculate_total_steps(arguments_dict): """ Get number of in-progress steps in importing process, as shown in the UI. """ - return len(list(MigrationStep)) + # We subtract the BULK_MIGRATION_PREFIX + return len(list(MigrationStep)) - 1 + + +class _BulkMigrationTask(UserTask): + """ + Base class for bulk_migrate_from_modulestore + """ + + @staticmethod + def calculate_total_steps(arguments_dict): + """ + Get number of in-progress steps in importing process, as shown in the UI. + + There are steps that are general for all sources, but there are steps that are repeated in each source. + All of this is taken into account to make the sum + """ + sources_count = len(arguments_dict.get('sources_pks', 1)) + + # STAGING, PARSING, IMPORTING_ASSETS, IMPORTING_STRUCTURE, MAPPING_OLD_TO_NEW, UNSTAGING + steps_repeated_count = 6 + + return ( + # All migration steps and subtract the BULK_MIGRATION_PREFIX + len(list(MigrationStep)) - 1 + # We don't want to count these steps again, they will be counted in the operation below. + - steps_repeated_count + # Each source repeats all the `steps_repeated_count` + + steps_repeated_count * sources_count + ) @dataclass(frozen=True) @@ -157,44 +191,34 @@ def should_fork_strategy(self) -> bool: return self.repeat_handling_strategy is RepeatHandlingStrategy.Fork -@shared_task(base=_MigrationTask, bind=True) -# Note: The decorator @set_code_owner_attribute cannot be used here because the UserTaskMixin -# does stack inspection and can't handle additional decorators. -def migrate_from_modulestore( - self: _MigrationTask, - *, - user_id: int, +@dataclass() +class _MigrationSourceData: + """ + Data related to a ModulestoreSource + """ + source: ModulestoreSource + source_root_usage_key: UsageKey + source_version: str | None + migration: ModulestoreMigration + + +def _validate_input( + status: UserTaskStatus, source_pk: int, - target_package_pk: int, - target_library_key: str, - target_collection_pk: int, repeat_handling_strategy: str, preserve_url_slugs: bool, composition_level: str, - forward_source_to_target: bool, -) -> None: + target_package: LearningPackage, + target_collection: Collection | None, +) -> _MigrationSourceData | None: """ - Import a course or legacy library into a learning package. - - Currently, the target learning package must be associated with a V2 content library, but that - restriction may be loosened in the future as more types of learning packages are developed. + Validates and build the source data related to `source_pk` """ - # pylint: disable=too-many-statements - # This is a large function, but breaking it up futher would probably not - # make it any easier to understand. - - set_code_owner_attribute_from_module(__name__) - - status: UserTaskStatus = self.status - status.set_state(MigrationStep.VALIDATING_INPUT.value) try: source = ModulestoreSource.objects.get(pk=source_pk) - target_package = LearningPackage.objects.get(pk=target_package_pk) - target_library = get_library(LibraryLocatorV2.from_string(target_library_key)) - target_collection = Collection.objects.get(pk=target_collection_pk) if target_collection_pk else None - except (ObjectDoesNotExist, InvalidKeyError) as exc: + except (ObjectDoesNotExist) as exc: status.fail(str(exc)) - return + return None # The Model is used for Course and Legacy Library course_index = SplitModulestoreCourseIndex.objects.filter(course_id=source.key).first() @@ -209,7 +233,7 @@ def migrate_from_modulestore( f"Not a valid source context key: {source.key}. " "Source key must reference a course or a legacy library." ) - return + return None migration = ModulestoreMigration.objects.create( source=source, @@ -221,21 +245,36 @@ def migrate_from_modulestore( target_collection=target_collection, task_status=status, ) - status.increment_completed_steps() - status.set_state(MigrationStep.CANCELLING_OLD.value) + return _MigrationSourceData( + source=source, + source_root_usage_key=source_root_usage_key, + source_version=source_version, + migration=migration, + ) + + +def _cancel_old_tasks( + source_list: list[ModulestoreSource], + status: UserTaskStatus, + target_package: LearningPackage, + migration_ids_to_exclude: list[int], +) -> None: + """ + Cancel all migration tasks related to the user and the source list + """ # In order to prevent a user from accidentally starting a bunch of identical import tasks... migrations_to_cancel = ModulestoreMigration.objects.filter( - # get all Migration tasks by this user with the same source and target + # get all Migration tasks by this user with the same sources and target task_status__user=status.user, - source=source, + source__in=source_list, target=target_package, ).select_related('task_status').exclude( # (excluding that aren't running) task_status__state__in=(UserTaskStatus.CANCELED, UserTaskStatus.FAILED, UserTaskStatus.SUCCEEDED) ).exclude( - # (excluding this migration itself) - id=migration.id + # (excluding these migrations themselves) + id__in=migration_ids_to_exclude ) # ... and cancel their tasks and clean away their staged content. for migration_to_cancel in migrations_to_cancel: @@ -243,45 +282,41 @@ def migrate_from_modulestore( migration_to_cancel.task_status.cancel() if migration_to_cancel.staged_content: migration_to_cancel.staged_content.delete() - status.increment_completed_steps() - status.set_state(MigrationStep.LOADING) + +def _load_xblock( + status: UserTaskStatus, + usage_key: UsageKey, +) -> XBlock | None: + """ + Loads the Xblock for the given usage_key + """ try: - legacy_root = modulestore().get_item(source_root_usage_key) + xblock = modulestore().get_item(usage_key) except modulestore_exceptions.ItemNotFoundError as exc: - status.fail(f"Failed to load source item '{source_root_usage_key}' from ModuleStore: {exc}") - return - if not legacy_root: - status.fail(f"Could not find source item '{source_root_usage_key}' in ModuleStore") - return - status.increment_completed_steps() + status.fail(f"Failed to load source item '{usage_key}' from ModuleStore: {exc}") + return None + if not xblock: + status.fail(f"Could not find source item '{usage_key}' in ModuleStore") + return None + return xblock - status.set_state(MigrationStep.STAGING.value) - staged_content = staging_api.stage_xblock_temporarily( - block=legacy_root, - user_id=status.user.pk, - purpose=CONTENT_STAGING_PURPOSE_TEMPLATE.format(source_key=source.key), - ) - migration.staged_content = staged_content - status.increment_completed_steps() - status.set_state(MigrationStep.PARSING.value) - parser = etree.XMLParser(strip_cdata=False) - try: - root_node = etree.fromstring(staged_content.olx, parser=parser) - except etree.ParseError as exc: - status.fail(f"Failed to parse source OLX (from staged content with id = {staged_content.id}): {exc}") - status.increment_completed_steps() +def _import_assets(migration: ModulestoreMigration) -> dict[str, int]: + """ + Import the assets of the staged content to the migration target + """ + if migration.staged_content is None: + return {} - status.set_state(MigrationStep.IMPORTING_ASSETS.value) content_by_filename: dict[str, int] = {} now = datetime.now(tz=timezone.utc) - for staged_content_file_data in staging_api.get_staged_content_static_files(staged_content.id): + for staged_content_file_data in staging_api.get_staged_content_static_files(migration.staged_content.id): old_path = staged_content_file_data.filename - file_data = staging_api.get_staged_content_static_file_data(staged_content.id, old_path) + file_data = staging_api.get_staged_content_static_file_data(migration.staged_content.id, old_path) if not file_data: log.error( - f"Staged content {staged_content.id} included referenced file {old_path}, " + f"Staged content {migration.staged_content.id} included referenced file {old_path}, " "but no file data was found." ) continue @@ -294,10 +329,45 @@ def migrate_from_modulestore( data=file_data, created=now, ).id - status.increment_completed_steps() + return content_by_filename - status.set_state(MigrationStep.IMPORTING_STRUCTURE.value) +def _import_structure( + migration: ModulestoreMigration, + source_data: _MigrationSourceData, + target_library: libraries_api.ContentLibraryMetadata, + content_by_filename: dict[str, int], + root_node: XmlTree, + status: UserTaskStatus, +) -> tuple[t.Any, _MigratedNode]: + """ + Import the staged content structure into the target Learning Core library. + + Args: + migration (ModulestoreMigration): + The migration record representing the ongoing modulestore-to-learning-core migration. + source_data (_MigrationSourceData): + Data extracted from the legacy modulestore, including the source root usage key. + Use `_validate_input()` to generate this data. + target_library (libraries_api.ContentLibraryMetadata): + The target library where the new Learning Core content will be created. + content_by_filename (dict[str, int]): + A mapping between OLX file names and their associated file IDs in the staging area. + Use `_import_assets` to generate this content. + root_node (XmlTree): + The parsed XML tree representing the root of the staged OLX content. + status (UserTaskStatus): + The user task used to record progress and state updates throughout the import. + + Returns: + tuple[Any, _MigratedNode]: + A tuple containing: + - The first element (`change_log`): the bulk draft change log generated by + `authoring_api.bulk_draft_changes_for`, containing all the imported changes. + - The second element (`root_migrated_node`): a `_MigratedNode` object that + represents the mapping between the legacy root node and its newly created + Learning Core equivalent. + """ # "key" is locally unique across all PublishableEntities within # a given LearningPackage. # We use this mapping to ensure that we don't create duplicate @@ -313,13 +383,13 @@ def migrate_from_modulestore( migration_context = _MigrationContext( existing_source_to_target_keys=existing_source_to_target_keys, - target_package_id=target_package_pk, + target_package_id=migration.target.pk, target_library_key=target_library.key, - source_context_key=source_root_usage_key.course_key, + source_context_key=source_data.source_root_usage_key.course_key, content_by_filename=content_by_filename, - composition_level=CompositionLevel(composition_level), - repeat_handling_strategy=RepeatHandlingStrategy(repeat_handling_strategy), - preserve_url_slugs=preserve_url_slugs, + composition_level=CompositionLevel(migration.composition_level), + repeat_handling_strategy=RepeatHandlingStrategy(migration.repeat_handling_strategy), + preserve_url_slugs=migration.preserve_url_slugs, created_by=status.user_id, created_at=datetime.now(timezone.utc), ) @@ -330,6 +400,215 @@ def migrate_from_modulestore( source_node=root_node, ) change_log.save() + return change_log, root_migrated_node + + +def _forwarding_content(source_data: _MigrationSourceData) -> None: + """ + Forwarding legacy content to migrated content + """ + block_migrations = ModulestoreBlockMigration.objects.filter(overall_migration=source_data.migration) + block_sources_to_block_migrations = { + block_migration.source: block_migration for block_migration in block_migrations + } + for block_source, block_migration in block_sources_to_block_migrations.items(): + block_source.forwarded = block_migration + block_source.save() + + source_data.source.forwarded = source_data.migration + source_data.source.save() + + +def _populate_collection(user_id: int, migration: ModulestoreMigration) -> None: + """ + Assigning imported items to the specified collection in the migration + """ + if migration.target_collection is None: + return + + block_target_pks: list[int] = list( + ModulestoreBlockMigration.objects.filter( + overall_migration=migration + ).values_list('target_id', flat=True) + ) + if block_target_pks: + authoring_api.add_to_collection( + learning_package_id=migration.target.pk, + key=migration.target_collection.key, + entities_qset=PublishableEntity.objects.filter(id__in=block_target_pks), + created_by=user_id, + ) + else: + log.warning("No target entities found to add to collection") + + +def _create_collection(library_key: LibraryLocatorV2, title: str) -> Collection: + """ + Creates a collection in the given library + + If there's a collection with the same key, try again, adding the attempt number at the end. + The same is true for the title. + """ + key = slugify(title) + collection = None + attempt = 0 + created_at = strftime_localized(datetime.now(timezone.utc), DEFAULT_DATE_TIME_FORMAT) + description = f"{_('This collection contains content migrated from a legacy library on')}: {created_at}" + while not collection: + modified_key = key if attempt == 0 else key + '-' + str(attempt) + try: + # Add transaction here to avoid TransactionManagementError on retry + with transaction.atomic(): + collection = libraries_api.create_library_collection( + library_key=library_key, + collection_key=modified_key, + title=f"{title}{f'_{attempt}' if attempt > 0 else ''}", + description=description, + ) + except libraries_api.LibraryCollectionAlreadyExists as e: + attempt += 1 + return collection + + +@shared_task(base=_MigrationTask, bind=True) +# Note: The decorator @set_code_owner_attribute cannot be used here because the UserTaskMixin +# does stack inspection and can't handle additional decorators. +def migrate_from_modulestore( + self: _MigrationTask, + *, + user_id: int, + source_pk: int, + target_library_key: str, + target_collection_pk: int | None, + repeat_handling_strategy: str, + preserve_url_slugs: bool, + composition_level: str, + forward_source_to_target: bool, +) -> None: + """ + Import a single course or legacy library from modulestore into a V2 legacy library. + + This task performs the end-to-end migration for one legacy source (course or library), + including staging, parsing OLX, importing assets and structure, and assigning the + migrated content to the specified target library and collection. + + A new `UserTaskStatus` entry is created for each invocation of this task, meaning + that each migration runs independently with its own progress tracking and final + success or failure state. + + If the migration encounters an unrecoverable error at any step (for example, invalid + OLX, missing assets, or database constraints), the task is marked as **failed** and + the partial results are rolled back as necessary. The migration state can be queried + through the REST API endpoint `/api/modulestore_migrator/v1/migrations//`. + + Args: + self (_MigrationTask): + The Celery task instance that wraps the user task logic. + user_id (int): + The ID of the user initiating the migration. + source_pk (int): + Primary key of the modulestore source to migrate. + target_library_key (str): + Key of the target V2 library that will receive the imported content. + target_collection_pk (int | None): + Optional ID of a target collection to which imported content will be assigned. + repeat_handling_strategy (str): + Strategy for handling repeated imports (e.g., "skip", "update"). + preserve_url_slugs (bool): + Whether to preserve original XBlock URL slugs during import. + composition_level (str): + The structural level to migrate (e.g., component, unit, or section). + forward_source_to_target (bool): + Whether to forward legacy content references to the migrated content after import. + + See Also: + - `bulk_migrate_from_modulestore`: Multi-source batch migration equivalent. + - API docs: `/api/cms/v1/migrations/` for REST behavior and responses. + """ + + # pylint: disable=too-many-statements + # This is a large function, but breaking it up futher would probably not + # make it any easier to understand. + + set_code_owner_attribute_from_module(__name__) + status: UserTaskStatus = self.status + + # Validating input + status.set_state(MigrationStep.VALIDATING_INPUT.value) + try: + target_library = get_library(LibraryLocatorV2.from_string(target_library_key)) + if target_library.learning_package_id is None: + raise ValueError("Target library has no associated learning package.") + + target_package = LearningPackage.objects.get(pk=target_library.learning_package_id) + target_collection = Collection.objects.get(pk=target_collection_pk) if target_collection_pk else None + except (ObjectDoesNotExist, InvalidKeyError) as exc: + status.fail(str(exc)) + return + + source_data = _validate_input( + status, + source_pk, + repeat_handling_strategy, + preserve_url_slugs, + composition_level, + target_package, + target_collection, + ) + if source_data is None: + # Fail + return + + migration = source_data.migration + status.increment_completed_steps() + + # Cancelling old tasks + status.set_state(MigrationStep.CANCELLING_OLD.value) + _cancel_old_tasks([source_data.source], status, target_package, [migration.id]) + status.increment_completed_steps() + + # Loading `legacy_root` + status.set_state(MigrationStep.LOADING) + legacy_root = _load_xblock(status, source_data.source_root_usage_key) + if legacy_root is None: + # Fail + return + status.increment_completed_steps() + + # Staging legacy block + status.set_state(MigrationStep.STAGING.value) + staged_content = staging_api.stage_xblock_temporarily( + block=legacy_root, + user_id=status.user.pk, + purpose=CONTENT_STAGING_PURPOSE_TEMPLATE.format(source_key=source_data.source.key), + ) + migration.staged_content = staged_content + status.increment_completed_steps() + + # Parsing OLX + status.set_state(MigrationStep.PARSING.value) + parser = etree.XMLParser(strip_cdata=False) + try: + root_node = etree.fromstring(staged_content.olx, parser=parser) + except etree.ParseError as exc: + status.fail(f"Failed to parse source OLX (from staged content with id = {staged_content.id}): {exc}") + status.increment_completed_steps() + + # Importing assets of the legacy block + status.set_state(MigrationStep.IMPORTING_ASSETS.value) + content_by_filename = _import_assets(migration) + status.increment_completed_steps() + + # Importing structure of the legacy block + status.set_state(MigrationStep.IMPORTING_STRUCTURE.value) + change_log, root_migrated_node = _import_structure( + migration, + source_data, + target_library, + content_by_filename, + root_node, + status, + ) migration.change_log = change_log status.increment_completed_steps() @@ -339,43 +618,271 @@ def migrate_from_modulestore( _create_migration_artifacts_incrementally( root_migrated_node=root_migrated_node, - source=source, + source=source_data.source, migration=migration, status=status, ) - - block_migrations = ModulestoreBlockMigration.objects.filter(overall_migration=migration) status.increment_completed_steps() + # Forwarding legacy content to migrated content status.set_state(MigrationStep.FORWARDING.value) if forward_source_to_target: - block_sources_to_block_migrations = { - block_migration.source: block_migration for block_migration in block_migrations - } - for block_source, block_migration in block_sources_to_block_migrations.items(): - block_source.forwarded = block_migration - block_source.save() - - source.forwarded = migration - source.save() + _forwarding_content(source_data) status.increment_completed_steps() + # Populating the collection status.set_state(MigrationStep.POPULATING_COLLECTION.value) if target_collection: - block_target_pks: list[int] = list( - ModulestoreBlockMigration.objects.filter( - overall_migration=migration - ).values_list('target_id', flat=True) + _populate_collection(user_id, migration) + status.increment_completed_steps() + + +@shared_task(base=_BulkMigrationTask, bind=True) +# Note: The decorator @set_code_owner_attribute cannot be used here because the UserTaskMixin +# does stack inspection and can't handle additional decorators. +def bulk_migrate_from_modulestore( + self: _BulkMigrationTask, + *, + user_id: int, + sources_pks: list[int], + target_library_key: str, + target_collection_pks: list[int | None], + create_collections: bool = False, + repeat_handling_strategy: str, + preserve_url_slugs: bool, + composition_level: str, + forward_source_to_target: bool, +) -> None: + """ + Import multiple legacy courses or libraries into a single V2 library. + + This task performs the same logical steps as `migrate_from_modulestore`, but allows + batching several migrations together under **one single user task** (`UserTaskStatus`). + + Unlike running `migrate_from_modulestore` in a loop (which would create multiple + independent Celery tasks and separate statuses), the bulk migration maintains + **one unified status record** that tracks progress across all included sources. + This simplifies monitoring, since the client only needs to observe one task state. + + Each source item (course or library) still creates its own `ModulestoreMigration` + database record, but all of them share the same parent task (`UserTaskStatus`). + If any sub-migration fails (for example, due to invalid OLX or missing assets), + the bulk migration **marks the entire task as failed** — there is no partial success. + + Args: + self (_BulkMigrationTask): + The Celery task instance that wraps the user task logic. + user_id (int): + The ID of the user initiating the migration. + sources_pks (list[int]): + Primary keys of the legacy modulestore sources to migrate. + target_library_key (str): + Key of the V2 library that will receive the imported content. + target_collection_pks (list[int | None]): + Optional list of target collection IDs corresponding to each source. + create_collections (bool): + Whether to automatically create new collections when none exist. + repeat_handling_strategy (str): + Strategy to handle repeated imports of the same content. + preserve_url_slugs (bool): + Whether to preserve existing XBlock URL slugs during import. + composition_level (str): + Composition level at which content should be imported (e.g. course, section). + forward_source_to_target (bool): + Whether to forward legacy content to its migrated equivalent after import. + + See Also: + - `migrate_from_modulestore`: Single-source migration equivalent. + - API docs: `/api/cms/v1/migrations/bulk/` for REST behavior and responses. + """ + # pylint: disable=too-many-statements + # This is a large function, but breaking it up futher would probably not + # make it any easier to understand. + + set_code_owner_attribute_from_module(__name__) + status: UserTaskStatus = self.status + + # Validating input + status.set_state(MigrationStep.VALIDATING_INPUT.value) + target_collection_list: list[Collection | None] = [] + + try: + target_library_locator = LibraryLocatorV2.from_string(target_library_key) + target_library = get_library(target_library_locator) + if target_library.learning_package_id is None: + raise ValueError("Target library has no associated learning package.") + + target_package = LearningPackage.objects.get(pk=target_library.learning_package_id) + + if target_collection_pks: + for target_collection_pk in target_collection_pks: + target_collection_list.append( + Collection.objects.get(pk=target_collection_pk) if target_collection_pk else None + ) + except (ObjectDoesNotExist, InvalidKeyError, ValueError) as exc: + status.fail(str(exc)) + return + + source_data_list: list[_MigrationSourceData] = [] + + for i in range(len(sources_pks)): + source_data = _validate_input( + status, + sources_pks[i], + repeat_handling_strategy, + preserve_url_slugs, + composition_level, + target_package, + target_collection_list[i] if target_collection_list else None, ) - if block_target_pks: - authoring_api.add_to_collection( - learning_package_id=target_package_pk, - key=target_collection.key, - entities_qset=PublishableEntity.objects.filter(id__in=block_target_pks), - created_by=user_id, - ) - else: - log.warning("No target entities found to add to collection") + if source_data is None: + # Fail + return + + source_data_list.append(source_data) + + status.increment_completed_steps() + + # Cancelling old tasks + status.set_state(MigrationStep.CANCELLING_OLD.value) + _cancel_old_tasks( + [x.source for x in source_data_list], + status, + target_package, + [migration.id for migration in [x.migration for x in source_data_list]], + ) + status.increment_completed_steps() + + # Loading legacy blocks + status.set_state(MigrationStep.LOADING) + legacy_root_list: list[XBlock] = [] + for source_data in source_data_list: + legacy_root = _load_xblock(status, source_data.source_root_usage_key) + if legacy_root is None: + # Fail + return + legacy_root_list.append(legacy_root) + status.increment_completed_steps() + + for i, source_pk in enumerate(sources_pks): + source_data = source_data_list[i] + + # Start migration for `source_pk` + # Staging legacy blocks + status.set_state(f"{MigrationStep.STAGING.BULK_MIGRATION_PREFIX} ({source_pk}): {MigrationStep.STAGING.value}") + staged_content = staging_api.stage_xblock_temporarily( + block=legacy_root_list[i], + user_id=status.user.pk, + purpose=CONTENT_STAGING_PURPOSE_TEMPLATE.format(source_key=source_data.source.key), + ) + source_data.migration.staged_content = staged_content + status.increment_completed_steps() + + # Parsing OLX + status.set_state(f"{MigrationStep.STAGING.BULK_MIGRATION_PREFIX} ({source_pk}): {MigrationStep.PARSING.value}") + parser = etree.XMLParser(strip_cdata=False) + try: + root_node = etree.fromstring(staged_content.olx, parser=parser) + except etree.ParseError as exc: + status.fail(f"Failed to parse source OLX (from staged content with id = {staged_content.id}): {exc}") + status.increment_completed_steps() + + # Importing assets + status.set_state( + f"{MigrationStep.STAGING.BULK_MIGRATION_PREFIX} ({source_pk}): {MigrationStep.IMPORTING_ASSETS.value}" + ) + content_by_filename = _import_assets(source_data.migration) + status.increment_completed_steps() + + # Importing structure of the legacy block + status.set_state( + f"{MigrationStep.STAGING.BULK_MIGRATION_PREFIX} ({source_pk}): {MigrationStep.IMPORTING_STRUCTURE.value}" + ) + change_log, root_migrated_node = _import_structure( + source_data.migration, + source_data, + target_library, + content_by_filename, + root_node, + status, + ) + source_data.migration.change_log = change_log + status.increment_completed_steps() + + status.set_state( + f"{MigrationStep.STAGING.BULK_MIGRATION_PREFIX} ({source_pk}): {MigrationStep.UNSTAGING.value}" + ) + staged_content.delete() + status.increment_completed_steps() + + _create_migration_artifacts_incrementally( + root_migrated_node=root_migrated_node, + source=source_data.source, + migration=source_data.migration, + status=status, + source_pk=source_pk, + ) + status.increment_completed_steps() + + # Forwarding legacy content to migrated content + status.set_state(MigrationStep.FORWARDING.value) + if forward_source_to_target: + for source_data in source_data_list: + _forwarding_content(source_data) + status.increment_completed_steps() + + # Populating collections + status.set_state(MigrationStep.POPULATING_COLLECTION.value) + + # Used to check if the source has a previous migration in a V2 library collection + # It is placed here to avoid the circular import + from .api import get_migration_info + for i, source_data in enumerate(source_data_list): + migration = source_data.migration + + title = legacy_root_list[i].display_name + if migration.target_collection is None: + if not create_collections: + continue + + source_key = source_data.source.key + + if migration.repeat_handling_strategy == RepeatHandlingStrategy.Fork.value: + # Create a new collection when it is Fork + migration.target_collection = _create_collection(target_library_locator, title) + else: + # It is Skip or Update + # We need to verify if there is a previous migration with collection + # TODO: This only fetches the latest migration, if different migrations have been done + # on different V2 libraries, this could break the logic. + previous_migration = get_migration_info([source_key]) + if ( + source_key in previous_migration + and previous_migration[source_key].migrations__target_collection__key + ): + # Has previous migration with collection + try: + # Get the previous collection + previous_collection = authoring_api.get_collection( + target_package.id, + previous_migration[source_key].migrations__target_collection__key, + ) + + migration.target_collection = previous_collection + except Collection.DoesNotExist: + # The collection no longer exists or is being migrated to a different library. + # In that case, create a new collection independent of strategy + migration.target_collection = _create_collection(target_library_locator, title) + else: + # Create collection and save in migration + migration.target_collection = _create_collection(target_library_locator, title) + + _populate_collection(user_id, migration) + + ModulestoreMigration.objects.bulk_update( + [x.migration for x in source_data_list], + ["target_collection"], + ) status.increment_completed_steps() @@ -751,7 +1258,8 @@ def _create_migration_artifacts_incrementally( root_migrated_node: _MigratedNode, source: ModulestoreSource, migration: ModulestoreMigration, - status: UserTaskStatus + status: UserTaskStatus, + source_pk: int | None = None, ) -> None: """ Create ModulestoreBlockSource and ModulestoreBlockMigration objects incrementally. @@ -774,6 +1282,12 @@ def _create_migration_artifacts_incrementally( processed += 1 if processed % 10 == 0 or processed == total_nodes: - status.set_state( - f"{MigrationStep.MAPPING_OLD_TO_NEW.value} ({processed}/{total_nodes})" - ) + if source_pk: + status.set_state( + f"{MigrationStep.STAGING.BULK_MIGRATION_PREFIX} ({source_pk}): " + f"{MigrationStep.MAPPING_OLD_TO_NEW.value} ({processed}/{total_nodes})" + ) + else: + status.set_state( + f"{MigrationStep.MAPPING_OLD_TO_NEW.value} ({processed}/{total_nodes})" + ) diff --git a/cms/djangoapps/modulestore_migrator/tests/test_api.py b/cms/djangoapps/modulestore_migrator/tests/test_api.py index 13e7e7685d9c..6f4daee94bc5 100644 --- a/cms/djangoapps/modulestore_migrator/tests/test_api.py +++ b/cms/djangoapps/modulestore_migrator/tests/test_api.py @@ -67,6 +67,45 @@ def test_start_migration_to_library(self): assert modulestoremigration.task_status is not None assert modulestoremigration.task_status.user == user + def test_start_bulk_migration_to_library(self): + """ + Test that the API can start a bulk migration to a library. + """ + source = ModulestoreSourceFactory() + source_2 = ModulestoreSourceFactory() + user = UserFactory() + + api.start_bulk_migration_to_library( + user=user, + source_key_list=[source.key, source_2.key], + target_library_key=self.library_v2.library_key, + target_collection_slug_list=None, + composition_level=CompositionLevel.Component.value, + repeat_handling_strategy=RepeatHandlingStrategy.Skip.value, + preserve_url_slugs=True, + forward_source_to_target=False, + ) + + modulestoremigration = ModulestoreMigration.objects.get(source=source) + assert modulestoremigration.source.key == source.key + assert ( + modulestoremigration.composition_level == CompositionLevel.Component.value + ) + assert modulestoremigration.repeat_handling_strategy == RepeatHandlingStrategy.Skip.value + assert modulestoremigration.preserve_url_slugs is True + assert modulestoremigration.task_status is not None + assert modulestoremigration.task_status.user == user + + modulestoremigration_2 = ModulestoreMigration.objects.get(source=source_2) + assert modulestoremigration_2.source.key == source_2.key + assert ( + modulestoremigration_2.composition_level == CompositionLevel.Component.value + ) + assert modulestoremigration_2.repeat_handling_strategy == RepeatHandlingStrategy.Skip.value + assert modulestoremigration_2.preserve_url_slugs is True + assert modulestoremigration_2.task_status is not None + assert modulestoremigration_2.task_status.user == user + def test_start_migration_to_library_with_collection(self): """ Test that the API can start a migration to a library with a target collection. diff --git a/cms/djangoapps/modulestore_migrator/tests/test_tasks.py b/cms/djangoapps/modulestore_migrator/tests/test_tasks.py index 309877ae0d0b..242d7c6eec6d 100644 --- a/cms/djangoapps/modulestore_migrator/tests/test_tasks.py +++ b/cms/djangoapps/modulestore_migrator/tests/test_tasks.py @@ -14,7 +14,7 @@ from user_tasks.models import UserTaskArtifact from user_tasks.tasks import UserTaskStatus from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase -from xmodule.modulestore.tests.factories import CourseFactory +from xmodule.modulestore.tests.factories import CourseFactory, LibraryFactory from common.djangoapps.student.tests.factories import UserFactory from cms.djangoapps.modulestore_migrator.data import CompositionLevel, RepeatHandlingStrategy @@ -29,7 +29,9 @@ _MigratedNode, _MigrationContext, _MigrationTask, + _BulkMigrationTask, migrate_from_modulestore, + bulk_migrate_from_modulestore, MigrationStep, ) from openedx.core.djangoapps.content_libraries import api as lib_api @@ -48,24 +50,55 @@ def setUp(self): self.lib_key = LibraryLocatorV2.from_string( f"lib:{self.organization.short_name}:test-key" ) + self.lib_key_2 = LibraryLocatorV2.from_string( + f"lib:{self.organization.short_name}:test-key-2" + ) lib_api.create_library( org=self.organization, slug=self.lib_key.slug, title="Test Library", ) + lib_api.create_library( + org=self.organization, + slug=self.lib_key_2.slug, + title="Test Library 2", + ) self.library = lib_api.ContentLibrary.objects.get(slug=self.lib_key.slug) + self.library_2 = lib_api.ContentLibrary.objects.get(slug=self.lib_key_2.slug) self.learning_package = self.library.learning_package + self.learning_package_2 = self.library_2.learning_package self.course = CourseFactory( org=self.organization.short_name, course="TestCourse", run="TestRun", display_name="Test Course", ) + self.course_2 = CourseFactory( + org=self.organization.short_name, + course="TestCourse2", + run="TestRun2", + display_name="Test Course 2", + ) + self.legacy_library = LibraryFactory( + org=self.organization.short_name, + library="LegacyLibrary", + display_name="Legacy Library", + ) + self.legacy_library_2 = LibraryFactory( + org=self.organization.short_name, + library="LegacyLibrary2", + display_name="Legacy Library 2", + ) self.collection = Collection.objects.create( learning_package=self.learning_package, key="test_collection", title="Test Collection", ) + self.collection2 = Collection.objects.create( + learning_package=self.learning_package, + key="test_collection2", + title="Test Collection 2", + ) def _get_task_status_fail_message(self, status): """ @@ -272,7 +305,6 @@ def test_migrate_from_modulestore_invalid_source(self): kwargs={ "user_id": self.user.id, "source_pk": 999999, # Non-existent source - "target_package_pk": self.learning_package.id, "target_library_key": str(self.lib_key), "target_collection_pk": self.collection.id, "repeat_handling_strategy": RepeatHandlingStrategy.Skip.value, @@ -286,9 +318,30 @@ def test_migrate_from_modulestore_invalid_source(self): self.assertEqual(status.state, UserTaskStatus.FAILED) self.assertEqual(self._get_task_status_fail_message(status), "ModulestoreSource matching query does not exist.") - def test_migrate_from_modulestore_invalid_target_package(self): + def test_bulk_migrate_invalid_sources(self): + """ + Test bulk_migrate_from_modulestore with invalid source + """ + task = bulk_migrate_from_modulestore.apply_async( + kwargs={ + "user_id": self.user.id, + "sources_pks": [999999], # Non-existent source + "target_library_key": str(self.lib_key), + "target_collection_pks": [self.collection.id], + "repeat_handling_strategy": RepeatHandlingStrategy.Skip.value, + "preserve_url_slugs": True, + "composition_level": CompositionLevel.Unit.value, + "forward_source_to_target": False, + } + ) + + status = UserTaskStatus.objects.get(task_id=task.id) + self.assertEqual(status.state, UserTaskStatus.FAILED) + self.assertEqual(self._get_task_status_fail_message(status), "ModulestoreSource matching query does not exist.") + + def test_migrate_from_modulestore_invalid_collection(self): """ - Test migrate_from_modulestore with invalid target package + Test migrate_from_modulestore with invalid collection """ source = ModulestoreSource.objects.create( key=self.course.id, @@ -298,9 +351,8 @@ def test_migrate_from_modulestore_invalid_target_package(self): kwargs={ "user_id": self.user.id, "source_pk": source.id, - "target_package_pk": 999999, # Non-existent package "target_library_key": str(self.lib_key), - "target_collection_pk": self.collection.id, + "target_collection_pk": 999999, # Non-existent collection "repeat_handling_strategy": RepeatHandlingStrategy.Skip.value, "preserve_url_slugs": True, "composition_level": CompositionLevel.Unit.value, @@ -310,23 +362,22 @@ def test_migrate_from_modulestore_invalid_target_package(self): status = UserTaskStatus.objects.get(task_id=task.id) self.assertEqual(status.state, UserTaskStatus.FAILED) - self.assertEqual(self._get_task_status_fail_message(status), "LearningPackage matching query does not exist.") + self.assertEqual(self._get_task_status_fail_message(status), "Collection matching query does not exist.") - def test_migrate_from_modulestore_invalid_collection(self): + def test_bulk_migrate_invalid_collection(self): """ - Test migrate_from_modulestore with invalid collection + Test bulk_migrate_from_modulestore with invalid collection """ source = ModulestoreSource.objects.create( key=self.course.id, ) - task = migrate_from_modulestore.apply_async( + task = bulk_migrate_from_modulestore.apply_async( kwargs={ "user_id": self.user.id, - "source_pk": source.id, - "target_package_pk": self.learning_package.id, + "sources_pks": [source.id], "target_library_key": str(self.lib_key), - "target_collection_pk": 999999, # Non-existent collection + "target_collection_pks": [999999], # Non-existent collection "repeat_handling_strategy": RepeatHandlingStrategy.Skip.value, "preserve_url_slugs": True, "composition_level": CompositionLevel.Unit.value, @@ -343,7 +394,17 @@ def test_migration_task_calculate_total_steps(self): Test _MigrationTask.calculate_total_steps returns correct count """ total_steps = _MigrationTask.calculate_total_steps({}) - expected_steps = len(list(MigrationStep)) + expected_steps = len(list(MigrationStep)) - 1 + self.assertEqual(total_steps, expected_steps) + + def test_bulk_migration_task_calculate_total_steps(self): + """ + Test _BulkMigrationTask.calculate_total_steps returns correct count + """ + total_steps = _BulkMigrationTask.calculate_total_steps({ + "sources_pks": [1, 2, 3, 4], + }) + expected_steps = len(list(MigrationStep)) - 1 + 6 * 3 self.assertEqual(total_steps, expected_steps) def test_migrate_component_success(self): @@ -1247,7 +1308,6 @@ def test_migrate_from_modulestore_success_course(self): kwargs={ "user_id": self.user.id, "source_pk": source.id, - "target_package_pk": self.learning_package.id, "target_library_key": str(self.lib_key), "target_collection_pk": self.collection.id, "repeat_handling_strategy": RepeatHandlingStrategy.Skip.value, @@ -1266,6 +1326,338 @@ def test_migrate_from_modulestore_success_course(self): self.assertEqual(migration.composition_level, CompositionLevel.Unit.value) self.assertEqual(migration.repeat_handling_strategy, RepeatHandlingStrategy.Skip.value) + def test_bulk_migrate_success_courses(self): + """ + Test successful bulk migration from courses to library + """ + source_1 = ModulestoreSource.objects.create(key=self.course.id) + source_2 = ModulestoreSource.objects.create(key=self.course_2.id) + + task = bulk_migrate_from_modulestore.apply_async( + kwargs={ + "user_id": self.user.id, + "sources_pks": [source_1.id, source_2.id], + "target_library_key": str(self.lib_key), + "target_collection_pks": [self.collection.id, self.collection2.id], + "repeat_handling_strategy": RepeatHandlingStrategy.Skip.value, + "preserve_url_slugs": True, + "composition_level": CompositionLevel.Unit.value, + "forward_source_to_target": False, + } + ) + + status = UserTaskStatus.objects.get(task_id=task.id) + self.assertEqual(status.state, UserTaskStatus.SUCCEEDED) + + migration = ModulestoreMigration.objects.get( + source=source_1.id, target=self.learning_package + ) + self.assertEqual(migration.composition_level, CompositionLevel.Unit.value) + self.assertEqual(migration.repeat_handling_strategy, RepeatHandlingStrategy.Skip.value) + + migration_2 = ModulestoreMigration.objects.get( + source=source_2.id, target=self.learning_package + ) + self.assertEqual(migration_2.composition_level, CompositionLevel.Unit.value) + self.assertEqual(migration_2.repeat_handling_strategy, RepeatHandlingStrategy.Skip.value) + + def test_migrate_from_modulestore_success_legacy_library(self): + """ + Test successful migration from legacy library to V2 library + """ + source = ModulestoreSource.objects.create(key=self.legacy_library.location.library_key) + + task = migrate_from_modulestore.apply_async( + kwargs={ + "user_id": self.user.id, + "source_pk": source.id, + "target_library_key": str(self.lib_key), + "target_collection_pk": self.collection.id, + "repeat_handling_strategy": RepeatHandlingStrategy.Skip.value, + "preserve_url_slugs": True, + "composition_level": CompositionLevel.Unit.value, + "forward_source_to_target": False, + } + ) + + status = UserTaskStatus.objects.get(task_id=task.id) + self.assertEqual(status.state, UserTaskStatus.SUCCEEDED) + + migration = ModulestoreMigration.objects.get( + source=source, target=self.learning_package + ) + self.assertEqual(migration.composition_level, CompositionLevel.Unit.value) + self.assertEqual(migration.repeat_handling_strategy, RepeatHandlingStrategy.Skip.value) + + def test_bulk_migrate_success_legacy_libraries(self): + """ + Test successful bulk migration from legacy libraries to V2 library + """ + source = ModulestoreSource.objects.create(key=self.legacy_library.location.library_key) + source_2 = ModulestoreSource.objects.create(key=self.legacy_library_2.location.library_key) + + task = bulk_migrate_from_modulestore.apply_async( + kwargs={ + "user_id": self.user.id, + "sources_pks": [source.id, source_2.id], + "target_library_key": str(self.lib_key), + "target_collection_pks": [self.collection.id, self.collection2.id], + "repeat_handling_strategy": RepeatHandlingStrategy.Skip.value, + "preserve_url_slugs": True, + "composition_level": CompositionLevel.Unit.value, + "forward_source_to_target": False, + } + ) + + status = UserTaskStatus.objects.get(task_id=task.id) + self.assertEqual(status.state, UserTaskStatus.SUCCEEDED) + + migration = ModulestoreMigration.objects.get( + source=source, target=self.learning_package + ) + self.assertEqual(migration.composition_level, CompositionLevel.Unit.value) + self.assertEqual(migration.repeat_handling_strategy, RepeatHandlingStrategy.Skip.value) + + migration_2 = ModulestoreMigration.objects.get( + source=source_2, target=self.learning_package + ) + self.assertEqual(migration_2.composition_level, CompositionLevel.Unit.value) + self.assertEqual(migration_2.repeat_handling_strategy, RepeatHandlingStrategy.Skip.value) + + def test_bulk_migrate_create_collections(self): + """ + Test successful bulk migration from legacy libraries to V2 library with create collections + """ + source = ModulestoreSource.objects.create(key=self.legacy_library.location.library_key) + source_2 = ModulestoreSource.objects.create(key=self.legacy_library_2.location.library_key) + + task = bulk_migrate_from_modulestore.apply_async( + kwargs={ + "user_id": self.user.id, + "sources_pks": [source.id, source_2.id], + "target_library_key": str(self.lib_key), + "target_collection_pks": [], + "create_collections": True, + "repeat_handling_strategy": RepeatHandlingStrategy.Skip.value, + "preserve_url_slugs": True, + "composition_level": CompositionLevel.Unit.value, + "forward_source_to_target": False, + } + ) + + status = UserTaskStatus.objects.get(task_id=task.id) + self.assertEqual(status.state, UserTaskStatus.SUCCEEDED) + + migration = ModulestoreMigration.objects.get( + source=source, target=self.learning_package + ) + self.assertEqual(migration.composition_level, CompositionLevel.Unit.value) + self.assertEqual(migration.repeat_handling_strategy, RepeatHandlingStrategy.Skip.value) + self.assertEqual(migration.target_collection.title, self.legacy_library.display_name) + + migration_2 = ModulestoreMigration.objects.get( + source=source_2, target=self.learning_package + ) + self.assertEqual(migration_2.composition_level, CompositionLevel.Unit.value) + self.assertEqual(migration_2.repeat_handling_strategy, RepeatHandlingStrategy.Skip.value) + self.assertEqual(migration_2.target_collection.title, self.legacy_library_2.display_name) + + @ddt.data( + RepeatHandlingStrategy.Skip, + RepeatHandlingStrategy.Update, + ) + def test_bulk_migrate_use_previous_collection_on_skip_and_update(self, repeat_handling_strategy): + """ + Test successful bulk migration from legacy libraries to V2 library using previous collection + """ + source = ModulestoreSource.objects.create(key=self.legacy_library.location.library_key) + + task = bulk_migrate_from_modulestore.apply_async( + kwargs={ + "user_id": self.user.id, + "sources_pks": [source.id], + "target_library_key": str(self.lib_key), + "target_collection_pks": [], + "create_collections": True, + "repeat_handling_strategy": repeat_handling_strategy.value, + "preserve_url_slugs": True, + "composition_level": CompositionLevel.Unit.value, + "forward_source_to_target": False, + } + ) + + status = UserTaskStatus.objects.get(task_id=task.id) + self.assertEqual(status.state, UserTaskStatus.SUCCEEDED) + + migration = ModulestoreMigration.objects.get( + source=source, target=self.learning_package + ) + self.assertEqual(migration.composition_level, CompositionLevel.Unit.value) + self.assertEqual(migration.repeat_handling_strategy, repeat_handling_strategy.value) + self.assertEqual(migration.target_collection.title, self.legacy_library.display_name) + + # Migrate again and check that the migration uses the previos collection + previous_collection = migration.target_collection + + task = bulk_migrate_from_modulestore.apply_async( + kwargs={ + "user_id": self.user.id, + "sources_pks": [source.id], + "target_library_key": str(self.lib_key), + "target_collection_pks": [], + "create_collections": True, + "repeat_handling_strategy": repeat_handling_strategy.value, + "preserve_url_slugs": True, + "composition_level": CompositionLevel.Unit.value, + "forward_source_to_target": False, + } + ) + status = UserTaskStatus.objects.get(task_id=task.id) + self.assertEqual(status.state, UserTaskStatus.SUCCEEDED) + + migrations = ModulestoreMigration.objects.filter( + source=source, target=self.learning_package + ) + + for migration in migrations: + self.assertEqual(migration.composition_level, CompositionLevel.Unit.value) + self.assertEqual(migration.repeat_handling_strategy, repeat_handling_strategy.value) + self.assertEqual(migration.target_collection.title, self.legacy_library.display_name) + self.assertEqual(migration.target_collection.id, previous_collection.id) + + @ddt.data( + RepeatHandlingStrategy.Skip, + RepeatHandlingStrategy.Update, + ) + def test_bulk_migrate_create_collection_in_different_learning_packages(self, repeat_handling_strategy): + """ + Test successful bulk migration from legacy libraries to different V2 libraries + """ + source = ModulestoreSource.objects.create(key=self.legacy_library.location.library_key) + + task = bulk_migrate_from_modulestore.apply_async( + kwargs={ + "user_id": self.user.id, + "sources_pks": [source.id], + "target_library_key": str(self.lib_key), + "target_collection_pks": [], + "create_collections": True, + "repeat_handling_strategy": repeat_handling_strategy.value, + "preserve_url_slugs": True, + "composition_level": CompositionLevel.Unit.value, + "forward_source_to_target": False, + } + ) + + status = UserTaskStatus.objects.get(task_id=task.id) + self.assertEqual(status.state, UserTaskStatus.SUCCEEDED) + + migration = ModulestoreMigration.objects.get( + source=source, target=self.learning_package + ) + self.assertEqual(migration.composition_level, CompositionLevel.Unit.value) + self.assertEqual(migration.repeat_handling_strategy, repeat_handling_strategy.value) + self.assertEqual(migration.target_collection.title, self.legacy_library.display_name) + + # Migrate again in other V2 library, verify that the collections are different + previous_collection = migration.target_collection + + task = bulk_migrate_from_modulestore.apply_async( + kwargs={ + "user_id": self.user.id, + "sources_pks": [source.id], + "target_library_key": str(self.lib_key_2), + "target_collection_pks": [], + "create_collections": True, + "repeat_handling_strategy": repeat_handling_strategy.value, + "preserve_url_slugs": True, + "composition_level": CompositionLevel.Unit.value, + "forward_source_to_target": False, + } + ) + status = UserTaskStatus.objects.get(task_id=task.id) + self.assertEqual(status.state, UserTaskStatus.SUCCEEDED) + + migration = ModulestoreMigration.objects.get( + source=source, target=self.learning_package + ) + self.assertEqual(migration.composition_level, CompositionLevel.Unit.value) + self.assertEqual(migration.repeat_handling_strategy, repeat_handling_strategy.value) + self.assertEqual(migration.target_collection.title, self.legacy_library.display_name) + self.assertEqual(migration.target_collection.id, previous_collection.id) + + migration = ModulestoreMigration.objects.get( + source=source, target=self.learning_package_2 + ) + self.assertEqual(migration.composition_level, CompositionLevel.Unit.value) + self.assertEqual(migration.repeat_handling_strategy, repeat_handling_strategy.value) + self.assertEqual(migration.target_collection.title, self.legacy_library.display_name) + self.assertNotEqual(migration.target_collection.id, previous_collection.id) + + def test_bulk_migrate_create_a_new_collection_on_fork(self): + """ + Test successful bulk migration from legacy libraries to V2 library using previous collection + """ + source = ModulestoreSource.objects.create(key=self.legacy_library.location.library_key) + + task = bulk_migrate_from_modulestore.apply_async( + kwargs={ + "user_id": self.user.id, + "sources_pks": [source.id], + "target_library_key": str(self.lib_key), + "target_collection_pks": [], + "create_collections": True, + "repeat_handling_strategy": RepeatHandlingStrategy.Fork.value, + "preserve_url_slugs": True, + "composition_level": CompositionLevel.Unit.value, + "forward_source_to_target": False, + } + ) + + status = UserTaskStatus.objects.get(task_id=task.id) + self.assertEqual(status.state, UserTaskStatus.SUCCEEDED) + + migration = ModulestoreMigration.objects.get( + source=source, target=self.learning_package + ) + self.assertEqual(migration.composition_level, CompositionLevel.Unit.value) + self.assertEqual(migration.repeat_handling_strategy, RepeatHandlingStrategy.Fork.value) + self.assertEqual(migration.target_collection.title, self.legacy_library.display_name) + previous_collection = migration.target_collection + + # Migrate again and check that it creates a new collection + task = bulk_migrate_from_modulestore.apply_async( + kwargs={ + "user_id": self.user.id, + "sources_pks": [source.id], + "target_library_key": str(self.lib_key), + "target_collection_pks": [], + "create_collections": True, + "repeat_handling_strategy": RepeatHandlingStrategy.Fork.value, + "preserve_url_slugs": True, + "composition_level": CompositionLevel.Unit.value, + "forward_source_to_target": False, + } + ) + status = UserTaskStatus.objects.get(task_id=task.id) + self.assertEqual(status.state, UserTaskStatus.SUCCEEDED) + + migrations = ModulestoreMigration.objects.filter( + source=source, target=self.learning_package + ) + + # First migration + self.assertEqual(migrations[0].composition_level, CompositionLevel.Unit.value) + self.assertEqual(migrations[0].repeat_handling_strategy, RepeatHandlingStrategy.Fork.value) + self.assertEqual(migrations[0].target_collection.title, self.legacy_library.display_name) + self.assertEqual(migrations[0].target_collection.id, previous_collection.id) + + # Second migration + self.assertEqual(migrations[1].composition_level, CompositionLevel.Unit.value) + self.assertEqual(migrations[1].repeat_handling_strategy, RepeatHandlingStrategy.Fork.value) + self.assertEqual(migrations[1].target_collection.title, f"{self.legacy_library.display_name}_1") + self.assertNotEqual(migrations[1].target_collection.id, previous_collection.id) + def test_migrate_from_modulestore_library_validation_failure(self): """ Test migration from legacy library fails when modulestore content doesn't exist @@ -1278,7 +1670,6 @@ def test_migrate_from_modulestore_library_validation_failure(self): kwargs={ "user_id": self.user.id, "source_pk": source.id, - "target_package_pk": self.learning_package.id, "target_library_key": str(self.lib_key), "target_collection_pk": None, "repeat_handling_strategy": RepeatHandlingStrategy.Update.value, @@ -1309,7 +1700,6 @@ def test_migrate_from_modulestore_invalid_source_key_type(self): kwargs={ "user_id": self.user.id, "source_pk": source.id, - "target_package_pk": self.learning_package.id, "target_library_key": str(self.lib_key), "target_collection_pk": self.collection.id, "repeat_handling_strategy": RepeatHandlingStrategy.Skip.value, @@ -1326,6 +1716,33 @@ def test_migrate_from_modulestore_invalid_source_key_type(self): f"Not a valid source context key: {invalid_key}. Source key must reference a course or a legacy library." ) + def test_bulk_migrate_invalid_source_key_type(self): + """ + Test bulk migration with invalid source key type + """ + invalid_key = LibraryLocatorV2.from_string("lib:testorg:invalid") + source = ModulestoreSource.objects.create(key=invalid_key) + + task = bulk_migrate_from_modulestore.apply_async( + kwargs={ + "user_id": self.user.id, + "sources_pks": [source.id], + "target_library_key": str(self.lib_key), + "target_collection_pks": [self.collection.id], + "repeat_handling_strategy": RepeatHandlingStrategy.Skip.value, + "preserve_url_slugs": True, + "composition_level": CompositionLevel.Unit.value, + "forward_source_to_target": False, + } + ) + + status = UserTaskStatus.objects.get(task_id=task.id) + self.assertEqual(status.state, UserTaskStatus.FAILED) + self.assertEqual( + self._get_task_status_fail_message(status), + f"Not a valid source context key: {invalid_key}. Source key must reference a course or a legacy library." + ) + def test_migrate_from_modulestore_nonexistent_modulestore_item(self): """ Test migration when modulestore item doesn't exist @@ -1339,7 +1756,6 @@ def test_migrate_from_modulestore_nonexistent_modulestore_item(self): kwargs={ "user_id": self.user.id, "source_pk": source.id, - "target_package_pk": self.learning_package.id, "target_library_key": str(self.lib_key), "target_collection_pk": self.collection.id, "repeat_handling_strategy": RepeatHandlingStrategy.Skip.value, @@ -1357,6 +1773,36 @@ def test_migrate_from_modulestore_nonexistent_modulestore_item(self): "from ModuleStore: course-v1:NonExistent+Course+Run+branch@draft-branch" ) + def test_bulk_migrate_nonexistent_modulestore_item(self): + """ + Test bulk migration when modulestore item doesn't exist + """ + nonexistent_course_key = CourseKey.from_string( + "course-v1:NonExistent+Course+Run" + ) + source = ModulestoreSource.objects.create(key=nonexistent_course_key) + + task = bulk_migrate_from_modulestore.apply_async( + kwargs={ + "user_id": self.user.id, + "sources_pks": [source.id], + "target_library_key": str(self.lib_key), + "target_collection_pks": [self.collection.id], + "repeat_handling_strategy": RepeatHandlingStrategy.Skip.value, + "preserve_url_slugs": True, + "composition_level": CompositionLevel.Unit.value, + "forward_source_to_target": False, + } + ) + + status = UserTaskStatus.objects.get(task_id=task.id) + self.assertEqual(status.state, UserTaskStatus.FAILED) + self.assertEqual( + self._get_task_status_fail_message(status), + "Failed to load source item 'block-v1:NonExistent+Course+Run+type@course+block@course' " + "from ModuleStore: course-v1:NonExistent+Course+Run+branch@draft-branch" + ) + def test_migrate_from_modulestore_task_status_progression(self): """Test that task status progresses through expected steps""" source = ModulestoreSource.objects.create(key=self.course.id) @@ -1365,7 +1811,6 @@ def test_migrate_from_modulestore_task_status_progression(self): kwargs={ "user_id": self.user.id, "source_pk": source.id, - "target_package_pk": self.learning_package.id, "target_library_key": str(self.lib_key), "target_collection_pk": self.collection.id, "repeat_handling_strategy": RepeatHandlingStrategy.Skip.value, @@ -1396,7 +1841,6 @@ def test_migrate_from_modulestore_multiple_users_no_interference(self): kwargs={ "user_id": self.user.id, "source_pk": source.id, - "target_package_pk": self.learning_package.id, "target_library_key": str(self.lib_key), "target_collection_pk": self.collection.id, "repeat_handling_strategy": RepeatHandlingStrategy.Skip.value, @@ -1410,7 +1854,6 @@ def test_migrate_from_modulestore_multiple_users_no_interference(self): kwargs={ "user_id": other_user.id, "source_pk": source.id, - "target_package_pk": self.learning_package.id, "target_library_key": str(self.lib_key), "target_collection_pk": self.collection.id, "repeat_handling_strategy": RepeatHandlingStrategy.Skip.value, @@ -1428,3 +1871,45 @@ def test_migrate_from_modulestore_multiple_users_no_interference(self): # The first task should not be cancelled since it's from a different user self.assertNotEqual(status1.state, UserTaskStatus.CANCELED) + + def test_bulk_migrate_multiple_users_no_interference(self): + """ + Test that migrations by different users don't interfere with each other + """ + source = ModulestoreSource.objects.create(key=self.course.id) + other_user = UserFactory() + + task1 = bulk_migrate_from_modulestore.apply_async( + kwargs={ + "user_id": self.user.id, + "sources_pks": [source.id], + "target_library_key": str(self.lib_key), + "target_collection_pks": [self.collection.id], + "repeat_handling_strategy": RepeatHandlingStrategy.Skip.value, + "preserve_url_slugs": True, + "composition_level": CompositionLevel.Unit.value, + "forward_source_to_target": False, + } + ) + + task2 = bulk_migrate_from_modulestore.apply_async( + kwargs={ + "user_id": other_user.id, + "sources_pks": [source.id], + "target_library_key": str(self.lib_key), + "target_collection_pks": [self.collection.id], + "repeat_handling_strategy": RepeatHandlingStrategy.Skip.value, + "preserve_url_slugs": True, + "composition_level": CompositionLevel.Unit.value, + "forward_source_to_target": False, + } + ) + + status1 = UserTaskStatus.objects.get(task_id=task1.id) + status2 = UserTaskStatus.objects.get(task_id=task2.id) + + self.assertEqual(status1.user, self.user) + self.assertEqual(status2.user, other_user) + + # The first task should not be cancelled since it's from a different user + self.assertNotEqual(status1.state, UserTaskStatus.CANCELED) From 5523072086200d70dd8d6bc10ef4598b1f808970 Mon Sep 17 00:00:00 2001 From: iloveagent57 <2307986+iloveagent57@users.noreply.github.com> Date: Fri, 10 Oct 2025 13:22:26 +0000 Subject: [PATCH 016/351] feat: Upgrade Python dependency edx-enterprise feat: add an endpoint to create a customer admin user Commit generated by workflow `openedx/edx-platform/.github/workflows/upgrade-one-python-dependency.yml@refs/heads/master` --- requirements/constraints.txt | 2 +- requirements/edx/base.txt | 2 +- requirements/edx/development.txt | 2 +- requirements/edx/doc.txt | 2 +- requirements/edx/testing.txt | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/requirements/constraints.txt b/requirements/constraints.txt index f78de7473150..b4fa1f404a4f 100644 --- a/requirements/constraints.txt +++ b/requirements/constraints.txt @@ -42,7 +42,7 @@ django-stubs<6 # The team that owns this package will manually bump this package rather than having it pulled in automatically. # This is to allow them to better control its deployment and to do it in a process that works better # for them. -edx-enterprise==6.4.4 +edx-enterprise==6.5.0 # Date: 2023-07-26 # Our legacy Sass code is incompatible with anything except this ancient libsass version. diff --git a/requirements/edx/base.txt b/requirements/edx/base.txt index 730460139839..bf633e192747 100644 --- a/requirements/edx/base.txt +++ b/requirements/edx/base.txt @@ -473,7 +473,7 @@ edx-drf-extensions==10.6.0 # edxval # enterprise-integrated-channels # openedx-learning -edx-enterprise==6.4.4 +edx-enterprise==6.5.0 # via # -c requirements/constraints.txt # -r requirements/edx/kernel.in diff --git a/requirements/edx/development.txt b/requirements/edx/development.txt index 9df3845c2e62..8732a8c8649a 100644 --- a/requirements/edx/development.txt +++ b/requirements/edx/development.txt @@ -747,7 +747,7 @@ edx-drf-extensions==10.6.0 # edxval # enterprise-integrated-channels # openedx-learning -edx-enterprise==6.4.4 +edx-enterprise==6.5.0 # via # -c requirements/constraints.txt # -r requirements/edx/doc.txt diff --git a/requirements/edx/doc.txt b/requirements/edx/doc.txt index faba8969f10b..dd626ceaeba5 100644 --- a/requirements/edx/doc.txt +++ b/requirements/edx/doc.txt @@ -557,7 +557,7 @@ edx-drf-extensions==10.6.0 # edxval # enterprise-integrated-channels # openedx-learning -edx-enterprise==6.4.4 +edx-enterprise==6.5.0 # via # -c requirements/constraints.txt # -r requirements/edx/base.txt diff --git a/requirements/edx/testing.txt b/requirements/edx/testing.txt index 7de977c71684..9e4960d6edf2 100644 --- a/requirements/edx/testing.txt +++ b/requirements/edx/testing.txt @@ -578,7 +578,7 @@ edx-drf-extensions==10.6.0 # edxval # enterprise-integrated-channels # openedx-learning -edx-enterprise==6.4.4 +edx-enterprise==6.5.0 # via # -c requirements/constraints.txt # -r requirements/edx/base.txt From 2268c5a92f0bda2e7742863387fd619744cbb8c9 Mon Sep 17 00:00:00 2001 From: Feanil Patel Date: Fri, 10 Oct 2025 11:09:24 -0400 Subject: [PATCH 017/351] fix: Remove templates which are never used. The code that renders these was removed some time ago but this got missed I guess. This cleanup is part of https://github.com/openedx/edx-platform/issues/36108 and https://github.com/openedx/studio-frontend/issues/381 --- cms/templates/accessibility.html | 37 ---------------- cms/templates/checklists.html | 75 -------------------------------- 2 files changed, 112 deletions(-) delete mode 100644 cms/templates/accessibility.html delete mode 100644 cms/templates/checklists.html diff --git a/cms/templates/accessibility.html b/cms/templates/accessibility.html deleted file mode 100644 index bb7a13f11441..000000000000 --- a/cms/templates/accessibility.html +++ /dev/null @@ -1,37 +0,0 @@ -<%page expression_filter="h"/> -<%inherit file="base.html" /> -<%def name="online_help_token()"><% return "accessibility" %> -<%! - from django.urls import reverse - from django.utils.translation import gettext as _ - from openedx.core.djangolib.markup import HTML, Text - from openedx.core.djangolib.js_utils import js_escaped_string, dump_js_escaped_json -%> -<%block name="title">${_("Studio Accessibility Policy")} -<%block name="bodyclass">is-signedin not-signedin view-accessibility - -<%namespace name='static' file='static_content.html'/> - -<%block name="header_extras"> - % if not settings.STUDIO_FRONTEND_CONTAINER_URL: - - - % endif - - -<%block name="content"> - -
    -
    -
    -
    - <%static:studiofrontend entry="accessibilityPolicy"> - { - "lang": "${language_code | n, js_escaped_string}" - } - -
    -
    -
    - - diff --git a/cms/templates/checklists.html b/cms/templates/checklists.html deleted file mode 100644 index e1859f38f70b..000000000000 --- a/cms/templates/checklists.html +++ /dev/null @@ -1,75 +0,0 @@ -<%page expression_filter="h"/> -<%inherit file="base.html" /> -<%def name="online_help_token()"><% return "files" %> -<%! - from cms.djangoapps.contentstore import utils - from cms.djangoapps.contentstore.config.waffle_utils import should_show_checklists_quality - from django.urls import reverse - from django.utils.translation import gettext as _ - from openedx.core.djangolib.markup import HTML, Text - from openedx.core.djangolib.js_utils import js_escaped_string, dump_js_escaped_json - from common.djangoapps.util.course import has_certificates_enabled -%> -<%block name="title">${_("Checklists")} -<%block name="bodyclass">is-signedin course view-checklists - -<%namespace name='static' file='static_content.html'/> - -<%block name="header_extras"> - % if not settings.STUDIO_FRONTEND_CONTAINER_URL: - - - % endif - - -<%block name="content"> - -
    -
    -

    - ${_("Tools")} - - ${_("Checklists")} -

    -
    -
    - -
    -
    - <%static:studiofrontend entry="courseHealthCheck"> - - <% - course_key = str(context_course.id) - certificates_url = '' - if has_certificates_enabled(context_course): - certificates_url = utils.reverse_course_url('certificates_list_handler', course_key) - %> - { - "lang": "${language_code | n, js_escaped_string}", - "course": { - "id": "${context_course.id | n, js_escaped_string}", - "name": "${context_course.display_name_with_default | n, js_escaped_string}", - "is_course_self_paced": ${context_course.self_paced | n, dump_js_escaped_json}, - "url_name": "${context_course.location.block_id | n, js_escaped_string}", - "org": "${context_course.location.org | n, js_escaped_string}", - "num": "${context_course.location.course | n, js_escaped_string}", - "display_course_number": "${context_course.display_coursenumber | n, js_escaped_string}", - "revision": "${context_course.location.revision | n, js_escaped_string}" - }, - "help_tokens": { - "files": "${get_online_help_info(online_help_token())['doc_url'] | n, js_escaped_string}" - }, - "enable_quality": ${should_show_checklists_quality(context_course.id) | n, dump_js_escaped_json}, - "links": { - "certificates": ${certificates_url | n, dump_js_escaped_json}, - "course_outline": ${utils.reverse_course_url('course_handler', course_key) | n, dump_js_escaped_json}, - "course_updates": ${utils.reverse_course_url('course_info_handler', course_key) | n, dump_js_escaped_json}, - "grading_policy": ${utils.reverse_course_url('grading_handler', course_key) | n, dump_js_escaped_json}, - "settings": ${utils.reverse_course_url('settings_handler', course_key) | n, dump_js_escaped_json}, - "proctored_exam_settings": ${mfe_proctored_exam_settings_url | n, dump_js_escaped_json} - } - } - -
    -
    - - From da2daf255eb6e03d0935e48bb565849d04a29e4f Mon Sep 17 00:00:00 2001 From: Raymond Zhou <56318341+rayzhou-bit@users.noreply.github.com> Date: Tue, 14 Oct 2025 18:32:48 -0400 Subject: [PATCH 018/351] feat: future proof artifact uploads (#37464) --- .github/workflows/unit-tests.yml | 10 +++++----- openedx/core/process_warnings.py | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index 05e5f47d1aae..9ffc2f1639f1 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -123,26 +123,26 @@ jobs: shell: bash run: | cd test_root/log - mv pytest_warnings.json pytest_warnings_${{ matrix.shard_name }}.json + mv pytest_warnings.json pytest_warnings_${{ matrix.shard_name }}_${{ matrix.python-version }}_${{ matrix.django-version }}_${{ matrix.mongo-version }}_${{ matrix.os-version }}.json - name: save pytest warnings json file if: success() uses: actions/upload-artifact@v4 with: - name: pytest-warnings-json-${{ matrix.shard_name }} + name: pytest-warnings-json-${{ matrix.shard_name }}-${{ matrix.python-version }}-${{ matrix.django-version }}-${{ matrix.mongo-version }}-${{ matrix.os-version }} path: | test_root/log/pytest_warnings*.json overwrite: true - name: Renaming coverage data file run: | - mv reports/.coverage reports/${{ matrix.shard_name }}.coverage + mv reports/.coverage reports/${{ matrix.shard_name }}_${{ matrix.python-version }}_${{ matrix.django-version }}_${{ matrix.mongo-version }}_${{ matrix.os-version }}.coverage - name: Upload coverage uses: actions/upload-artifact@v4 with: - name: coverage-${{ matrix.shard_name }} - path: reports/${{ matrix.shard_name }}.coverage + name: coverage-${{ matrix.shard_name }}-${{ matrix.python-version }}-${{ matrix.django-version }}-${{ matrix.mongo-version }}-${{ matrix.os-version }} + path: reports/${{ matrix.shard_name }}_${{ matrix.python-version }}_${{ matrix.django-version }}_${{ matrix.mongo-version }}_${{ matrix.os-version }}.coverage overwrite: true collect-and-verify: diff --git a/openedx/core/process_warnings.py b/openedx/core/process_warnings.py index 6f695fa27796..712f0f1ea8f5 100644 --- a/openedx/core/process_warnings.py +++ b/openedx/core/process_warnings.py @@ -91,7 +91,7 @@ def read_warning_data(dir_path): # TODO(jinder): currently this is hard-coded in, maybe create a constants file with info # THINK(jinder): but creating file for one constant seems overkill warnings_file_name_regex = ( - r"pytest_warnings_?[\w-]*\.json" # noqa pylint: disable=W1401 + r"pytest_warnings_?[\w.-]*\.json" # noqa pylint: disable=W1401 ) # iterate through files_in_dir and see if they match our know file name pattern From 7e1a17a70735c5e72a3d447a3258c7845378a7ee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Chris=20Ch=C3=A1vez?= Date: Wed, 15 Oct 2025 19:16:51 -0500 Subject: [PATCH 019/351] feat: Multiple updates to handle children upstream info [FC-0097] (#37433) * Which edX user roles will this change impact? "Developer"". * Added `upstream_ready_to_sync_children_info` in `ContainerChildrenSerializer` * Now, the `ContainerChildrenView` can return the `upstream_ready_to_sync_children_info` * Update the child info in `UpstreamLink._check_children_ready_to_sync` --- .../rest_api/v1/serializers/vertical_block.py | 15 +++++ .../rest_api/v1/views/course_index.py | 22 +++++++ .../v1/views/tests/test_vertical_block.py | 60 ++++++++++++++++--- .../v2/views/tests/test_downstreams.py | 2 + cms/lib/xblock/upstream_sync.py | 3 + 5 files changed, 93 insertions(+), 9 deletions(-) diff --git a/cms/djangoapps/contentstore/rest_api/v1/serializers/vertical_block.py b/cms/djangoapps/contentstore/rest_api/v1/serializers/vertical_block.py index 2283036faf24..87a40304faef 100644 --- a/cms/djangoapps/contentstore/rest_api/v1/serializers/vertical_block.py +++ b/cms/djangoapps/contentstore/rest_api/v1/serializers/vertical_block.py @@ -177,7 +177,22 @@ class ContainerChildrenSerializer(serializers.Serializer): Serializer for representing a vertical container with state and children. """ + class UpstreamReadyToSyncChildrenInfoSerializer(serializers.Serializer): + """ + Serializer used for the `upstream_ready_to_sync_children_info` field + """ + id = serializers.CharField() + name = serializers.CharField() + upstream = serializers.CharField() + block_type = serializers.CharField() + is_modified = serializers.BooleanField() + children = ContainerChildSerializer(many=True) is_published = serializers.BooleanField() can_paste_component = serializers.BooleanField() display_name = serializers.CharField() + upstream_ready_to_sync_children_info = UpstreamReadyToSyncChildrenInfoSerializer( + many=True, + required=False, + help_text="List of dictionaries describing upstream child components readiness to sync." + ) diff --git a/cms/djangoapps/contentstore/rest_api/v1/views/course_index.py b/cms/djangoapps/contentstore/rest_api/v1/views/course_index.py index 7ab14028cd50..f392c47a67a1 100644 --- a/cms/djangoapps/contentstore/rest_api/v1/views/course_index.py +++ b/cms/djangoapps/contentstore/rest_api/v1/views/course_index.py @@ -8,6 +8,7 @@ from rest_framework.request import Request from rest_framework.response import Response from rest_framework.views import APIView +from rest_framework.fields import BooleanField from cms.djangoapps.contentstore.config.waffle import CUSTOM_RELATIVE_DATES from cms.djangoapps.contentstore.rest_api.v1.mixins import ContainerHandlerMixin @@ -129,6 +130,11 @@ class ContainerChildrenView(APIView, ContainerHandlerMixin): apidocs.ParameterLocation.PATH, description="Container usage key", ), + apidocs.string_parameter( + "get_upstream_info", + apidocs.ParameterLocation.QUERY, + description="Gets the info of all ready to sync children", + ), ], responses={ 200: ContainerChildrenSerializer, @@ -210,6 +216,7 @@ def get(self, request: Request, usage_key_string: str): "version_available": 49, "error_message": null, "ready_to_sync": true, + "is_ready_to_sync_individually": true, }, "actions": { "can_copy": true, @@ -231,11 +238,19 @@ def get(self, request: Request, usage_key_string: str): "is_published": false, "can_paste_component": true, "display_name": "Vertical block 1" + "upstream_ready_to_sync_children_info": [{ + "name": "Text", + "upstream": "lb:org:mylib:html:abcd", + 'block_type': "html", + 'is_modified': true, + 'id': "block-v1:org+101+101+type@html+block@3e3fa1f88adb4a108cd14e9002143690", + }] } ``` """ usage_key = self.get_object(usage_key_string) current_xblock = get_xblock(usage_key, request.user) + get_upstream_info = BooleanField().to_internal_value(request.GET.get("get_upstream_info", False)) is_course = current_xblock.scope_ids.usage_id.context_key.is_course with modulestore().bulk_operations(usage_key.course_key): @@ -274,10 +289,17 @@ def get(self, request: Request, usage_key_string: str): except ItemNotFoundError: logging.error('Could not find any changes for block [%s]', usage_key) + upstream_ready_to_sync_children_info = [] + if current_xblock.upstream and get_upstream_info: + upstream_link = UpstreamLink.get_for_block(current_xblock) + upstream_link_data = upstream_link.to_json(include_child_info=True) + upstream_ready_to_sync_children_info = upstream_link_data["ready_to_sync_children"] + container_data = { "children": children, "is_published": is_published, "can_paste_component": is_course, + "upstream_ready_to_sync_children_info": upstream_ready_to_sync_children_info, "display_name": current_xblock.display_name_with_default, } serializer = ContainerChildrenSerializer(container_data) diff --git a/cms/djangoapps/contentstore/rest_api/v1/views/tests/test_vertical_block.py b/cms/djangoapps/contentstore/rest_api/v1/views/tests/test_vertical_block.py index 5dea51b91d71..22f0cd0d54d4 100644 --- a/cms/djangoapps/contentstore/rest_api/v1/views/tests/test_vertical_block.py +++ b/cms/djangoapps/contentstore/rest_api/v1/views/tests/test_vertical_block.py @@ -11,6 +11,7 @@ from cms.djangoapps.contentstore.tests.utils import CourseTestCase from openedx.core.djangoapps.content_tagging.toggles import DISABLE_TAGGING_FEATURE +from openedx.core.djangoapps.content_libraries.tests import ContentLibrariesRestApiTest from xmodule.partitions.partitions import ( ENROLLMENT_TRACK_PARTITION_ID, Group, @@ -27,7 +28,7 @@ ) # lint-amnesty, pylint: disable=wrong-import-order -class BaseXBlockContainer(CourseTestCase): +class BaseXBlockContainer(CourseTestCase, ContentLibrariesRestApiTest): """ Base xBlock container handler. @@ -48,6 +49,20 @@ def setup_xblock(self): This method creates XBlock objects representing a course structure with chapters, sequentials, verticals and others. """ + self.lib = self._create_library( + slug="containers", + title="Container Test Library", + description="Units and more", + ) + self.unit = self._create_container(self.lib["id"], "unit", display_name="Unit", slug=None) + self.html_block = self._add_block_to_library(self.lib["id"], "html", "Html1", can_stand_alone=False) + self._set_library_block_olx( + self.html_block["id"], + 'updated content upstream 1' + ) + # Set version of html to 2 + self._publish_library_block(self.html_block["id"]) + self.chapter = self.create_block( parent=self.course.location, category="chapter", @@ -60,7 +75,13 @@ def setup_xblock(self): display_name="Lesson 1", ) - self.vertical = self.create_block(self.sequential.location, "vertical", "Unit") + self.vertical = self.create_block( + self.sequential.location, + "vertical", + "Unit", + upstream=self.unit["id"], + upstream_version=1, + ) self.html_unit_first = self.create_block( parent=self.vertical.location, @@ -72,8 +93,8 @@ def setup_xblock(self): parent=self.vertical.location, category="html", display_name="Html Content 2", - upstream="lb:FakeOrg:FakeLib:html:FakeBlock", - upstream_version=5, + upstream=self.html_block["id"], + upstream_version=1, ) def create_block(self, parent, category, display_name, **kwargs): @@ -209,6 +230,27 @@ def test_success_response(self): self.assertFalse(data["is_published"]) self.assertTrue(data["can_paste_component"]) self.assertEqual(data["display_name"], "Unit") + self.assertEqual(data["upstream_ready_to_sync_children_info"], []) + + def test_success_response_with_upstream_info(self): + """ + Check that endpoint returns valid response data using `get_upstream_info` query param + """ + url = self.get_reverse_url(self.vertical.location) + response = self.client.get(f"{url}?get_upstream_info=true") + self.assertEqual(response.status_code, status.HTTP_200_OK) + data = response.json() + self.assertEqual(len(data["children"]), 2) + self.assertFalse(data["is_published"]) + self.assertTrue(data["can_paste_component"]) + self.assertEqual(data["display_name"], "Unit") + self.assertEqual(data["upstream_ready_to_sync_children_info"], [{ + "id": str(self.html_unit_second.usage_key), + "upstream": self.html_block["id"], + "block_type": "html", + "is_modified": False, + "name": "Html Content 2", + }]) def test_xblock_is_published(self): """ @@ -275,12 +317,12 @@ def test_children_content(self): "can_manage_tags": True, }, "upstream_link": { - "upstream_ref": "lb:FakeOrg:FakeLib:html:FakeBlock", - "version_synced": 5, - "version_available": None, + "upstream_ref": self.html_block["id"], + "version_synced": 1, + "version_available": 2, "version_declined": None, - "error_message": "Linked upstream library block was not found in the system", - "ready_to_sync": False, + "error_message": None, + "ready_to_sync": True, "has_top_level_parent": False, "is_modified": False, }, diff --git a/cms/djangoapps/contentstore/rest_api/v2/views/tests/test_downstreams.py b/cms/djangoapps/contentstore/rest_api/v2/views/tests/test_downstreams.py index c6f24496f241..cf3838ac3719 100644 --- a/cms/djangoapps/contentstore/rest_api/v2/views/tests/test_downstreams.py +++ b/cms/djangoapps/contentstore/rest_api/v2/views/tests/test_downstreams.py @@ -643,6 +643,8 @@ def test_200_single_upstream_container(self): self.assertDictEqual(data['ready_to_sync_children'][0], { 'name': html_block.display_name, 'upstream': str(self.html_lib_id_2), + 'block_type': 'html', + 'is_modified': False, 'id': str(html_block.usage_key), }) diff --git a/cms/lib/xblock/upstream_sync.py b/cms/lib/xblock/upstream_sync.py index 5e812f7035f5..78d14d01c246 100644 --- a/cms/lib/xblock/upstream_sync.py +++ b/cms/lib/xblock/upstream_sync.py @@ -121,6 +121,8 @@ def _check_children_ready_to_sync(self, xblock_downstream: XBlock, return_fast: child_info.append({ 'name': child.display_name, 'upstream': getattr(child, 'upstream', None), + 'block_type': child.usage_key.block_type, + 'is_modified': child_upstream_link.is_modified, 'id': str(child.usage_key), }) if return_fast: @@ -180,6 +182,7 @@ def to_json(self, include_child_info=False) -> dict[str, t.Any]: **asdict(self), "ready_to_sync": self.ready_to_sync, "upstream_link": self.upstream_link, + "is_ready_to_sync_individually": self.is_ready_to_sync_individually, } if ( include_child_info From 415c969ad37d576d03a7509a535006c34f120682 Mon Sep 17 00:00:00 2001 From: Nathan Sprenkle Date: Thu, 16 Oct 2025 10:31:40 -0400 Subject: [PATCH 020/351] chore: bump edxval to 3.1.0 (#37490) --- requirements/edx/base.txt | 2 +- requirements/edx/development.txt | 2 +- requirements/edx/doc.txt | 2 +- requirements/edx/testing.txt | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/requirements/edx/base.txt b/requirements/edx/base.txt index 6cb36c1986fb..6e6809d20237 100644 --- a/requirements/edx/base.txt +++ b/requirements/edx/base.txt @@ -553,7 +553,7 @@ edx-when==3.0.0 # via # -r requirements/edx/kernel.in # edx-proctoring -edxval==3.0.0 +edxval==3.1.0 # via -r requirements/edx/kernel.in elasticsearch==7.9.1 # via diff --git a/requirements/edx/development.txt b/requirements/edx/development.txt index a1e35b928940..89c2d3de5969 100644 --- a/requirements/edx/development.txt +++ b/requirements/edx/development.txt @@ -855,7 +855,7 @@ edx-when==3.0.0 # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt # edx-proctoring -edxval==3.0.0 +edxval==3.1.0 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt diff --git a/requirements/edx/doc.txt b/requirements/edx/doc.txt index cb280ad45e90..8d4ad9ecb2b7 100644 --- a/requirements/edx/doc.txt +++ b/requirements/edx/doc.txt @@ -639,7 +639,7 @@ edx-when==3.0.0 # via # -r requirements/edx/base.txt # edx-proctoring -edxval==3.0.0 +edxval==3.1.0 # via -r requirements/edx/base.txt elasticsearch==7.9.1 # via diff --git a/requirements/edx/testing.txt b/requirements/edx/testing.txt index 9a0d5e839de3..d7eab0980bf2 100644 --- a/requirements/edx/testing.txt +++ b/requirements/edx/testing.txt @@ -662,7 +662,7 @@ edx-when==3.0.0 # via # -r requirements/edx/base.txt # edx-proctoring -edxval==3.0.0 +edxval==3.1.0 # via -r requirements/edx/base.txt elasticsearch==7.9.1 # via From 5f94efcc89f5b90e1a2ffdce5fec57989a313a7f Mon Sep 17 00:00:00 2001 From: Nathan Sprenkle Date: Thu, 16 Oct 2025 10:31:40 -0400 Subject: [PATCH 021/351] chore: bump edxval to 3.1.0 (#37490) --- requirements/edx/base.txt | 2 +- requirements/edx/development.txt | 2 +- requirements/edx/doc.txt | 2 +- requirements/edx/testing.txt | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/requirements/edx/base.txt b/requirements/edx/base.txt index bf633e192747..194c693c9c78 100644 --- a/requirements/edx/base.txt +++ b/requirements/edx/base.txt @@ -553,7 +553,7 @@ edx-when==3.0.0 # via # -r requirements/edx/kernel.in # edx-proctoring -edxval==3.0.0 +edxval==3.1.0 # via -r requirements/edx/kernel.in elasticsearch==7.9.1 # via diff --git a/requirements/edx/development.txt b/requirements/edx/development.txt index 8732a8c8649a..c48117dbf35f 100644 --- a/requirements/edx/development.txt +++ b/requirements/edx/development.txt @@ -855,7 +855,7 @@ edx-when==3.0.0 # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt # edx-proctoring -edxval==3.0.0 +edxval==3.1.0 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt diff --git a/requirements/edx/doc.txt b/requirements/edx/doc.txt index dd626ceaeba5..06965ed48d0a 100644 --- a/requirements/edx/doc.txt +++ b/requirements/edx/doc.txt @@ -639,7 +639,7 @@ edx-when==3.0.0 # via # -r requirements/edx/base.txt # edx-proctoring -edxval==3.0.0 +edxval==3.1.0 # via -r requirements/edx/base.txt elasticsearch==7.9.1 # via diff --git a/requirements/edx/testing.txt b/requirements/edx/testing.txt index 9e4960d6edf2..b7bbce06c6e1 100644 --- a/requirements/edx/testing.txt +++ b/requirements/edx/testing.txt @@ -662,7 +662,7 @@ edx-when==3.0.0 # via # -r requirements/edx/base.txt # edx-proctoring -edxval==3.0.0 +edxval==3.1.0 # via -r requirements/edx/base.txt elasticsearch==7.9.1 # via From 9fc2441773c36f9b249f7a9b181db45bc6873a82 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 16 Oct 2025 16:54:01 +0000 Subject: [PATCH 022/351] build(deps): bump thollander/actions-comment-pull-request from 2 to 3 Bumps [thollander/actions-comment-pull-request](https://github.com/thollander/actions-comment-pull-request) from 2 to 3. - [Release notes](https://github.com/thollander/actions-comment-pull-request/releases) - [Commits](https://github.com/thollander/actions-comment-pull-request/compare/v2...v3) --- updated-dependencies: - dependency-name: thollander/actions-comment-pull-request dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/check-for-tutorial-prs.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/check-for-tutorial-prs.yml b/.github/workflows/check-for-tutorial-prs.yml index 1dc4d3860956..e3969a1920e9 100644 --- a/.github/workflows/check-for-tutorial-prs.yml +++ b/.github/workflows/check-for-tutorial-prs.yml @@ -26,7 +26,7 @@ jobs: uses: actions/checkout@v5 - name: Comment PR - uses: thollander/actions-comment-pull-request@v2 + uses: thollander/actions-comment-pull-request@v3 with: message: | Thank you for your pull request! Congratulations on completing the Open edX tutorial! A team member will be by to take a look shortly. From 1704a0807ae2adc82b4d48c1b5b2f88a88276951 Mon Sep 17 00:00:00 2001 From: Tony Busa Date: Thu, 16 Oct 2025 12:24:45 -0600 Subject: [PATCH 023/351] chore: remove karma-selenium-webdriver-launcher and unneeded browsers --- common/static/common/js/karma.common.conf.js | 28 +------------------- package-lock.json | 14 ---------- package.json | 1 - 3 files changed, 1 insertion(+), 42 deletions(-) diff --git a/common/static/common/js/karma.common.conf.js b/common/static/common/js/karma.common.conf.js index b49db1298d87..91a2d78085cb 100644 --- a/common/static/common/js/karma.common.conf.js +++ b/common/static/common/js/karma.common.conf.js @@ -285,7 +285,6 @@ function getBaseConfig(config, useRequireJs) { 'karma-chrome-launcher', 'karma-firefox-launcher', 'karma-spec-reporter', - 'karma-selenium-webdriver-launcher', 'karma-webpack', 'karma-sourcemap-loader', customPlugin @@ -339,33 +338,8 @@ function getBaseConfig(config, useRequireJs) { 'media.autoplay.enabled.user-gestures-needed': false, } }, - ChromeDocker: { - base: 'SeleniumWebdriver', - browserName: 'chrome', - getDriver: function() { - return new webdriver.Builder() - .forBrowser('chrome') - .usingServer('http://edx.devstack.chrome:4444/wd/hub') - .build(); - } - }, - FirefoxDocker: { - base: 'SeleniumWebdriver', - browserName: 'firefox', - getDriver: function() { - var options = new firefox.Options(), - profile = new firefox.Profile(); - profile.setPreference('focusmanager.testmode', true); - options.setProfile(profile); - return new webdriver.Builder() - .forBrowser('firefox') - .usingServer('http://edx.devstack.firefox:4444/wd/hub') - .setFirefoxOptions(options) - .build(); - } - } }, - + // Continuous Integration mode // if true, Karma captures browsers, runs the tests and exits singleRun: config.singleRun, diff --git a/package-lock.json b/package-lock.json index 0da387590a61..1c0c3f397850 100644 --- a/package-lock.json +++ b/package-lock.json @@ -89,7 +89,6 @@ "karma-jasmine-html-reporter": "0.2.2", "karma-junit-reporter": "2.0.1", "karma-requirejs": "1.1.0", - "karma-selenium-webdriver-launcher": "github:openedx/karma-selenium-webdriver-launcher#0.0.4-openedx.0", "karma-sourcemap-loader": "0.4.0", "karma-spec-reporter": "0.0.20", "karma-webpack": "^5.0.1", @@ -12925,19 +12924,6 @@ "requirejs": "^2.1.0" } }, - "node_modules/karma-selenium-webdriver-launcher": { - "version": "0.0.4-openedx.0", - "resolved": "git+ssh://git@github.com/openedx/karma-selenium-webdriver-launcher.git#79cfdc5037eb8585dd3e584875e4343febb6d61f", - "dev": true, - "license": "MIT", - "dependencies": { - "q": "~0.9.6" - }, - "peerDependencies": { - "karma": ">=0.9", - "selenium-webdriver": ">=2.44.0" - } - }, "node_modules/karma-sourcemap-loader": { "version": "0.4.0", "resolved": "https://registry.npmjs.org/karma-sourcemap-loader/-/karma-sourcemap-loader-0.4.0.tgz", diff --git a/package.json b/package.json index d3e14e536056..0899b244bb22 100644 --- a/package.json +++ b/package.json @@ -114,7 +114,6 @@ "karma-jasmine-html-reporter": "0.2.2", "karma-junit-reporter": "2.0.1", "karma-requirejs": "1.1.0", - "karma-selenium-webdriver-launcher": "github:openedx/karma-selenium-webdriver-launcher#0.0.4-openedx.0", "karma-sourcemap-loader": "0.4.0", "karma-spec-reporter": "0.0.20", "karma-webpack": "^5.0.1", From e68eab9e5b865318f2f511c52f1552e47a253c17 Mon Sep 17 00:00:00 2001 From: Feanil Patel Date: Thu, 9 Oct 2025 11:20:27 -0400 Subject: [PATCH 024/351] feat: Drop the `legacy_studio.text_editor` flag. Remove the flag and update the code paths as if it's always set to true. --- cms/djangoapps/contentstore/toggles.py | 19 ------------------- cms/djangoapps/contentstore/utils.py | 11 ++++------- cms/static/js/views/pages/container.js | 10 ++++------ cms/templates/container.html | 4 +--- cms/templates/studio_xblock_wrapper.html | 4 +--- 5 files changed, 10 insertions(+), 38 deletions(-) diff --git a/cms/djangoapps/contentstore/toggles.py b/cms/djangoapps/contentstore/toggles.py index c287f8c4dbec..55ec0e4ff5dc 100644 --- a/cms/djangoapps/contentstore/toggles.py +++ b/cms/djangoapps/contentstore/toggles.py @@ -86,25 +86,6 @@ def exam_setting_view_enabled(course_key): return not LEGACY_STUDIO_EXAM_SETTINGS.is_enabled(course_key) -# .. toggle_name: legacy_studio.text_editor -# .. toggle_implementation: WaffleFlag -# .. toggle_default: False -# .. toggle_description: Temporarily fall back to the old Text component (a.k.a. html block) editor. -# .. toggle_use_cases: temporary -# .. toggle_creation_date: 2025-03-14 -# .. toggle_target_removal_date: 2025-09-14 -# .. toggle_tickets: https://github.com/openedx/edx-platform/issues/36275 -# .. toggle_warning: In Ulmo, this toggle will be removed. Only the new (React-based) experience will be available. -LEGACY_STUDIO_TEXT_EDITOR = CourseWaffleFlag("legacy_studio.text_editor", __name__) - - -def use_new_text_editor(course_key): - """ - Returns a boolean = true if new text editor is enabled - """ - return not LEGACY_STUDIO_TEXT_EDITOR.is_enabled(course_key) - - # .. toggle_name: legacy_studio.video_editor # .. toggle_implementation: WaffleFlag # .. toggle_default: False diff --git a/cms/djangoapps/contentstore/utils.py b/cms/djangoapps/contentstore/utils.py index c6861b134dd8..efd6905745b3 100644 --- a/cms/djangoapps/contentstore/utils.py +++ b/cms/djangoapps/contentstore/utils.py @@ -53,11 +53,9 @@ use_new_home_page, use_new_import_page, use_new_schedule_details_page, - use_new_text_editor, use_new_textbooks_page, use_new_unit_page, use_new_updates_page, - use_new_video_editor, use_new_video_uploads_page, ) from cms.djangoapps.models.settings.course_grading import CourseGradingModel @@ -289,11 +287,10 @@ def get_editor_page_base_url(course_locator) -> str: Gets course authoring microfrontend URL for links to the new base editors """ editor_url = None - if use_new_text_editor(course_locator) or use_new_video_editor(course_locator): - mfe_base_url = get_course_authoring_url(course_locator) - course_mfe_url = f'{mfe_base_url}/course/{course_locator}/editor' - if mfe_base_url: - editor_url = course_mfe_url + mfe_base_url = get_course_authoring_url(course_locator) + course_mfe_url = f'{mfe_base_url}/course/{course_locator}/editor' + if mfe_base_url: + editor_url = course_mfe_url return editor_url diff --git a/cms/static/js/views/pages/container.js b/cms/static/js/views/pages/container.js index d50f6b4bbe4a..f65f88e2811d 100644 --- a/cms/static/js/views/pages/container.js +++ b/cms/static/js/views/pages/container.js @@ -504,12 +504,11 @@ function($, _, Backbone, gettext, BasePage, if (!options || options.view !== 'visibility_view') { const primaryHeader = $(event.target).closest('.xblock-header-primary, .nav-actions'); - var useNewTextEditor = primaryHeader.attr('use-new-editor-text'), - useNewVideoEditor = primaryHeader.attr('use-new-editor-video'), + var useNewVideoEditor = primaryHeader.attr('use-new-editor-video'), useNewProblemEditor = primaryHeader.attr('use-new-editor-problem'), blockType = primaryHeader.attr('data-block-type'); - if((useNewTextEditor === 'True' && blockType === 'html') + if((blockType === 'html') || (useNewVideoEditor === 'True' && blockType === 'video') || (useNewProblemEditor === 'True' && blockType === 'problem') ) { @@ -1170,8 +1169,7 @@ function($, _, Backbone, gettext, BasePage, }, onNewXBlock: function(xblockElement, scrollOffset, is_duplicate, data) { - var useNewTextEditor = this.$('.xblock-header-primary').attr('use-new-editor-text'), - useNewVideoEditor = this.$('.xblock-header-primary').attr('use-new-editor-video'), + var useNewVideoEditor = this.$('.xblock-header-primary').attr('use-new-editor-video'), useVideoGalleryFlow = this.$('.xblock-header-primary').attr("use-video-gallery-flow"), useNewProblemEditor = this.$('.xblock-header-primary').attr('use-new-editor-problem'); @@ -1181,7 +1179,7 @@ function($, _, Backbone, gettext, BasePage, var blockType = data.locator.match(matchBlockTypeFromLocator); } // open mfe editors for new blocks only and not for content imported from libraries - if(!data.hasOwnProperty('upstreamRef') && ((useNewTextEditor === 'True' && blockType.includes('html')) + if(!data.hasOwnProperty('upstreamRef') && (blockType.includes('html') || (useNewVideoEditor === 'True' && blockType.includes('video')) || (useNewProblemEditor === 'True' && blockType.includes('problem'))) ){ diff --git a/cms/templates/container.html b/cms/templates/container.html index 1c56740ca9e3..cf523ada4c9d 100644 --- a/cms/templates/container.html +++ b/cms/templates/container.html @@ -13,7 +13,7 @@ from django.utils.translation import gettext as _ from cms.djangoapps.contentstore.helpers import xblock_studio_url, xblock_type_display_name -from cms.djangoapps.contentstore.toggles import use_new_text_editor, use_new_problem_editor, use_new_video_editor, use_video_gallery_flow +from cms.djangoapps.contentstore.toggles import use_new_problem_editor, use_new_video_editor, use_video_gallery_flow from cms.djangoapps.contentstore.utils import get_editor_page_base_url from openedx.core.djangolib.js_utils import ( dump_js_escaped_json, js_escaped_string @@ -115,7 +115,6 @@ <%block name="content"> <% -use_new_editor_text = use_new_text_editor(xblock_locator.course_key) use_new_editor_video = use_new_video_editor(xblock_locator.course_key) use_new_editor_problem = use_new_problem_editor(xblock_locator.course_key) use_new_video_gallery_flow = use_video_gallery_flow() @@ -172,7 +171,6 @@

    From 542b6f84ed6fa15604c3582c9cabec8e25604cdc Mon Sep 17 00:00:00 2001 From: Feanil Patel Date: Thu, 16 Oct 2025 12:59:49 -0400 Subject: [PATCH 029/351] build: Refactor GitHub Actions workflow for dependencies Updated to checkout the code first since not all workflows(merge_queue) will check have the PR_URL setting set. Then grab the shas from the relevant event payload and use those to get the list of affected files. --- .../check-consistent-dependencies.yml | 29 +++++++++++-------- 1 file changed, 17 insertions(+), 12 deletions(-) diff --git a/.github/workflows/check-consistent-dependencies.yml b/.github/workflows/check-consistent-dependencies.yml index 87706e5a09db..af18cba66340 100644 --- a/.github/workflows/check-consistent-dependencies.yml +++ b/.github/workflows/check-consistent-dependencies.yml @@ -19,25 +19,30 @@ jobs: runs-on: ubuntu-24.04 steps: + # Always checkout the code because we don't always have a PR url. + - uses: actions/checkout@v5 + # Only run remaining steps if there are changes to requirements/** + # We do this instead of using path based short-circuiting. + # see https://stackoverflow.com/questions/77996177/how-can-i-handle-a-required-check-that-isnt-always-triggered + # for some more details. - name: "Decide whether to short-circuit" - env: - GH_TOKEN: "${{ github.token }}" - PR_URL: "${{ github.event.pull_request.html_url }}" run: | - paths=$(gh pr diff "$PR_URL" --name-only) - echo $'Paths touched in PR:\n'"$paths" + if [[ "${{ github.event_name }}" == "pull_request" ]]; then + BASE_SHA="${{ github.event.pull_request.base.sha }}" + else + BASE_SHA="${{ github.event.merge_group.base_sha }}" + fi + + # Fetch the base sha so we can compare to it. It's not checked out by + # default. + git fetch origin "$BASE_SHA" # The ^"? is because git may quote weird file paths - matched="$(echo "$paths" | grep -P '^"?((requirements/)|(scripts/.*?/requirements/))' || true)" - echo $'Relevant paths:\n'"$matched" - if [[ -n "$matched" ]]; then - echo "RELEVANT=true" >> "$GITHUB_ENV" + if git diff --name-only "$BASE_SHA" | grep -P '^"?((requirements/)|(scripts/.*?/requirements/))'; then + echo "RELEVANT=true" >> "$GITHUB_ENV" fi - - uses: actions/checkout@v5 - if: ${{ env.RELEVANT == 'true' }} - - uses: actions/setup-python@v5 if: ${{ env.RELEVANT == 'true' }} with: From ba56c0aa2d4e9ac8b060dfb403ade437b9a78597 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 17 Oct 2025 14:11:51 +0000 Subject: [PATCH 030/351] chore(deps): bump actions/setup-python from 5 to 6 Bumps [actions/setup-python](https://github.com/actions/setup-python) from 5 to 6. - [Release notes](https://github.com/actions/setup-python/releases) - [Commits](https://github.com/actions/setup-python/compare/v5...v6) --- updated-dependencies: - dependency-name: actions/setup-python dependency-version: '6' dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/check-consistent-dependencies.yml | 2 +- .github/workflows/check_python_dependencies.yml | 2 +- .github/workflows/ci-static-analysis.yml | 2 +- .github/workflows/compile-python-requirements.yml | 2 +- .github/workflows/js-tests.yml | 2 +- .github/workflows/lint-imports.yml | 2 +- .github/workflows/migrations-check.yml | 2 +- .github/workflows/pylint-checks.yml | 2 +- .github/workflows/quality-checks.yml | 2 +- .github/workflows/semgrep.yml | 2 +- .github/workflows/static-assets-check.yml | 2 +- .github/workflows/unit-tests.yml | 6 +++--- .github/workflows/units-test-scripts-structures-pruning.yml | 2 +- .github/workflows/units-test-scripts-user-retirement.yml | 2 +- .github/workflows/upgrade-one-python-dependency.yml | 2 +- 15 files changed, 17 insertions(+), 17 deletions(-) diff --git a/.github/workflows/check-consistent-dependencies.yml b/.github/workflows/check-consistent-dependencies.yml index af18cba66340..46f801179e84 100644 --- a/.github/workflows/check-consistent-dependencies.yml +++ b/.github/workflows/check-consistent-dependencies.yml @@ -43,7 +43,7 @@ jobs: echo "RELEVANT=true" >> "$GITHUB_ENV" fi - - uses: actions/setup-python@v5 + - uses: actions/setup-python@v6 if: ${{ env.RELEVANT == 'true' }} with: python-version: '3.11' diff --git a/.github/workflows/check_python_dependencies.yml b/.github/workflows/check_python_dependencies.yml index 64da3b985fef..f215880f0ef4 100644 --- a/.github/workflows/check_python_dependencies.yml +++ b/.github/workflows/check_python_dependencies.yml @@ -17,7 +17,7 @@ jobs: uses: actions/checkout@v5 - name: Set up Python - uses: actions/setup-python@v5 + uses: actions/setup-python@v6 with: python-version: ${{ matrix.python-version }} diff --git a/.github/workflows/ci-static-analysis.yml b/.github/workflows/ci-static-analysis.yml index 16af7aa69609..deb2853899f4 100644 --- a/.github/workflows/ci-static-analysis.yml +++ b/.github/workflows/ci-static-analysis.yml @@ -17,7 +17,7 @@ jobs: steps: - uses: actions/checkout@v5 - name: Set up Python - uses: actions/setup-python@v5 + uses: actions/setup-python@v6 with: python-version: ${{ matrix.python-version }} diff --git a/.github/workflows/compile-python-requirements.yml b/.github/workflows/compile-python-requirements.yml index 8673cc3c234c..9b4eb7f79753 100644 --- a/.github/workflows/compile-python-requirements.yml +++ b/.github/workflows/compile-python-requirements.yml @@ -24,7 +24,7 @@ jobs: ref: "${{ inputs.branch }}" - name: Set up Python environment - uses: actions/setup-python@v5 + uses: actions/setup-python@v6 with: python-version: "3.11" diff --git a/.github/workflows/js-tests.yml b/.github/workflows/js-tests.yml index 266e75278315..5ac3f5ab2d78 100644 --- a/.github/workflows/js-tests.yml +++ b/.github/workflows/js-tests.yml @@ -44,7 +44,7 @@ jobs: run: sudo apt-get update && sudo apt-get install libxmlsec1-dev ubuntu-restricted-extras xvfb - name: Setup Python - uses: actions/setup-python@v5 + uses: actions/setup-python@v6 with: python-version: ${{ matrix.python-version }} diff --git a/.github/workflows/lint-imports.yml b/.github/workflows/lint-imports.yml index a3d2129dd279..03034b465a40 100644 --- a/.github/workflows/lint-imports.yml +++ b/.github/workflows/lint-imports.yml @@ -17,7 +17,7 @@ jobs: uses: actions/checkout@v5 - name: Set up Python - uses: actions/setup-python@v5 + uses: actions/setup-python@v6 with: python-version: "3.11" diff --git a/.github/workflows/migrations-check.yml b/.github/workflows/migrations-check.yml index 966a6635f905..ffbc816cbee1 100644 --- a/.github/workflows/migrations-check.yml +++ b/.github/workflows/migrations-check.yml @@ -74,7 +74,7 @@ jobs: uses: actions/checkout@v5 - name: Setup Python ${{ matrix.python-version }} - uses: actions/setup-python@v5 + uses: actions/setup-python@v6 with: python-version: ${{ matrix.python-version }} diff --git a/.github/workflows/pylint-checks.yml b/.github/workflows/pylint-checks.yml index 124745cc5ea6..1d6944cc6671 100644 --- a/.github/workflows/pylint-checks.yml +++ b/.github/workflows/pylint-checks.yml @@ -38,7 +38,7 @@ jobs: run: sudo apt-get update && sudo apt-get install libxmlsec1-dev - name: Set up Python - uses: actions/setup-python@v5 + uses: actions/setup-python@v6 with: python-version: 3.11 diff --git a/.github/workflows/quality-checks.yml b/.github/workflows/quality-checks.yml index 32c576852304..b41d08df6608 100644 --- a/.github/workflows/quality-checks.yml +++ b/.github/workflows/quality-checks.yml @@ -31,7 +31,7 @@ jobs: run: sudo apt-get update && sudo apt-get install libxmlsec1-dev - name: Setup Python - uses: actions/setup-python@v5 + uses: actions/setup-python@v6 with: python-version: ${{ matrix.python-version }} diff --git a/.github/workflows/semgrep.yml b/.github/workflows/semgrep.yml index 8360ae4650d1..24bf5ed1c62c 100644 --- a/.github/workflows/semgrep.yml +++ b/.github/workflows/semgrep.yml @@ -27,7 +27,7 @@ jobs: with: fetch-depth: 1 - - uses: actions/setup-python@v5 + - uses: actions/setup-python@v6 with: python-version: "${{ matrix.python-version }}" diff --git a/.github/workflows/static-assets-check.yml b/.github/workflows/static-assets-check.yml index 4c67916431a7..c30ea404a6f1 100644 --- a/.github/workflows/static-assets-check.yml +++ b/.github/workflows/static-assets-check.yml @@ -39,7 +39,7 @@ jobs: uses: actions/checkout@v5 - name: Setup Python ${{ matrix.python-version }} - uses: actions/setup-python@v5 + uses: actions/setup-python@v6 with: python-version: ${{ matrix.python-version }} diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index 04ee71471110..7fba6b6b4cf1 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -90,7 +90,7 @@ jobs: mongodb-version: ${{ matrix.mongo-version }} - name: Setup Python - uses: actions/setup-python@v5 + uses: actions/setup-python@v6 with: python-version: ${{ matrix.python-version }} @@ -153,7 +153,7 @@ jobs: steps: - uses: actions/checkout@v5 - name: Setup Python - uses: actions/setup-python@v5 + uses: actions/setup-python@v6 with: python-version: 3.11 @@ -283,7 +283,7 @@ jobs: uses: actions/checkout@v5 - name: Setup Python ${{ matrix.python-version }} - uses: actions/setup-python@v5 + uses: actions/setup-python@v6 with: python-version: ${{ matrix.python-version }} diff --git a/.github/workflows/units-test-scripts-structures-pruning.yml b/.github/workflows/units-test-scripts-structures-pruning.yml index d6575abacbc1..9ebc069ac940 100644 --- a/.github/workflows/units-test-scripts-structures-pruning.yml +++ b/.github/workflows/units-test-scripts-structures-pruning.yml @@ -21,7 +21,7 @@ jobs: uses: actions/checkout@v5 - name: Set up Python - uses: actions/setup-python@v5 + uses: actions/setup-python@v6 with: python-version: ${{ matrix.python-version }} diff --git a/.github/workflows/units-test-scripts-user-retirement.yml b/.github/workflows/units-test-scripts-user-retirement.yml index d4051de6bdb4..2b28be5fe3dd 100644 --- a/.github/workflows/units-test-scripts-user-retirement.yml +++ b/.github/workflows/units-test-scripts-user-retirement.yml @@ -21,7 +21,7 @@ jobs: uses: actions/checkout@v5 - name: Set up Python - uses: actions/setup-python@v5 + uses: actions/setup-python@v6 with: python-version: ${{ matrix.python-version }} diff --git a/.github/workflows/upgrade-one-python-dependency.yml b/.github/workflows/upgrade-one-python-dependency.yml index 3f9678593c25..4b4bdccaf401 100644 --- a/.github/workflows/upgrade-one-python-dependency.yml +++ b/.github/workflows/upgrade-one-python-dependency.yml @@ -37,7 +37,7 @@ jobs: ref: "${{ inputs.branch }}" - name: Set up Python environment - uses: actions/setup-python@v5 + uses: actions/setup-python@v6 with: python-version: "3.11" From 0077058e37518a0c0b0abf41522bda12e32f9a5e Mon Sep 17 00:00:00 2001 From: Feanil Patel Date: Thu, 9 Oct 2025 13:21:26 -0400 Subject: [PATCH 031/351] feat!: Drop the legacy studio home page. This is the page that lists the courses in studio. This has been replaced by an MFE and the MFE has been on by default since Teak. BREAKING CHANGE: Setting the `legacy_studio.home` waffle flag will no longer work. The code will behave as if this is set to false showing the new studio authoring MFE experience for the course home page. This has been the default since Teak. --- .../v1/serializers/course_waffle_flags.py | 8 +++++-- .../contentstore/tests/test_course_listing.py | 12 ----------- cms/djangoapps/contentstore/tests/tests.py | 21 +++++++++---------- cms/djangoapps/contentstore/toggles.py | 19 ----------------- cms/djangoapps/contentstore/utils.py | 18 +++++++++------- cms/djangoapps/contentstore/views/course.py | 8 +------ cms/templates/widgets/header.html | 10 --------- 7 files changed, 27 insertions(+), 69 deletions(-) diff --git a/cms/djangoapps/contentstore/rest_api/v1/serializers/course_waffle_flags.py b/cms/djangoapps/contentstore/rest_api/v1/serializers/course_waffle_flags.py index 3efb7b6226d4..b3a833129f09 100644 --- a/cms/djangoapps/contentstore/rest_api/v1/serializers/course_waffle_flags.py +++ b/cms/djangoapps/contentstore/rest_api/v1/serializers/course_waffle_flags.py @@ -40,9 +40,13 @@ def get_course_key(self): def get_use_new_home_page(self, obj): """ - Method to get the use_new_home_page switch + Method to indicate whether we should use the new home page. + + This used to be based on a waffle flag but the flag is being removed so we + default it to true for now until we can remove the need for it from the consumers + of this serializer and the related APIs. """ - return toggles.use_new_home_page() + return True def get_use_new_custom_pages(self, obj): """ diff --git a/cms/djangoapps/contentstore/tests/test_course_listing.py b/cms/djangoapps/contentstore/tests/test_course_listing.py index e46b493b7b39..d256228228cb 100644 --- a/cms/djangoapps/contentstore/tests/test_course_listing.py +++ b/cms/djangoapps/contentstore/tests/test_course_listing.py @@ -8,12 +8,9 @@ import ddt from ccx_keys.locator import CCXLocator -from django.conf import settings from django.test import RequestFactory -from edx_toggles.toggles.testutils import override_waffle_flag from opaque_keys.edx.locations import CourseLocator -from cms.djangoapps.contentstore import toggles from cms.djangoapps.contentstore.tests.utils import AjaxEnabledTestClient from cms.djangoapps.contentstore.utils import delete_course from cms.djangoapps.contentstore.views.course import ( @@ -89,15 +86,6 @@ def tearDown(self): self.client.logout() ModuleStoreTestCase.tearDown(self) # pylint: disable=non-parent-method-called - @override_waffle_flag(toggles.LEGACY_STUDIO_HOME, True) - def test_empty_course_listing(self): - """ - Test on empty course listing, studio name is properly displayed - """ - message = f"Are you staff on an existing {settings.STUDIO_SHORT_NAME} course?" - response = self.client.get('/home') - self.assertContains(response, message) - def test_get_course_list(self): """ Test getting courses with new access group format e.g. 'instructor_edx.course.run' diff --git a/cms/djangoapps/contentstore/tests/tests.py b/cms/djangoapps/contentstore/tests/tests.py index 40b0f8ad4cbf..d151b1d58526 100644 --- a/cms/djangoapps/contentstore/tests/tests.py +++ b/cms/djangoapps/contentstore/tests/tests.py @@ -11,7 +11,7 @@ import datetime import time from unittest import mock -from urllib.parse import quote_plus +from urllib.parse import quote_plus, unquote from ddt import data, ddt, unpack from django.conf import settings @@ -24,6 +24,7 @@ from cms.djangoapps.contentstore import toggles from cms.djangoapps.contentstore.tests.test_course_settings import CourseTestCase from cms.djangoapps.contentstore.tests.utils import AjaxEnabledTestClient, parse_json, registration, user +from cms.djangoapps.contentstore.utils import get_studio_home_url from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase # lint-amnesty, pylint: disable=wrong-import-order from xmodule.modulestore.tests.factories import CourseFactory # lint-amnesty, pylint: disable=wrong-import-order @@ -114,12 +115,6 @@ def setUp(self): # clear the cache so ratelimiting won't affect these tests cache.clear() - def check_page_get(self, url, expected): - resp = self.client.get_html(url) - self.assertEqual(resp.status_code, expected) - return resp - - @override_waffle_flag(toggles.LEGACY_STUDIO_HOME, True) def test_private_pages_auth(self): """Make sure pages that do require login work.""" auth_pages = ( @@ -143,7 +138,9 @@ def test_private_pages_auth(self): print('Not logged in') for page in auth_pages: print(f"Checking '{page}'") - self.check_page_get(page, expected=302) + resp = self.client.get_html(page) + assert resp.status_code == 302 + assert resp.url == unquote(reverse("login", query={"next": page})) # Logged in should work. self.login(self.email, self.pw) @@ -151,10 +148,11 @@ def test_private_pages_auth(self): print('Logged in') for page in simple_auth_pages: print(f"Checking '{page}'") - self.check_page_get(page, expected=200) + resp = self.client.get_html(page) + assert resp.status_code == 302 + assert resp.url == get_studio_home_url() @override_settings(SESSION_INACTIVITY_TIMEOUT_IN_SECONDS=1) - @override_waffle_flag(toggles.LEGACY_STUDIO_HOME, True) def test_inactive_session_timeout(self): """ Verify that an inactive session times out and redirects to the @@ -168,7 +166,8 @@ def test_inactive_session_timeout(self): # make sure we can access courseware immediately course_url = '/home/' resp = self.client.get_html(course_url) - self.assertEqual(resp.status_code, 200) + assert resp.status_code == 302 + assert resp.url == get_studio_home_url() # then wait a bit and see if we get timed out time.sleep(2) diff --git a/cms/djangoapps/contentstore/toggles.py b/cms/djangoapps/contentstore/toggles.py index 55ec0e4ff5dc..de05a46ef30b 100644 --- a/cms/djangoapps/contentstore/toggles.py +++ b/cms/djangoapps/contentstore/toggles.py @@ -162,25 +162,6 @@ def individualize_anonymous_user_id(course_id): return INDIVIDUALIZE_ANONYMOUS_USER_ID.is_enabled(course_id) -# .. toggle_name: legacy_studio.home -# .. toggle_implementation: WaffleFlag -# .. toggle_default: False -# .. toggle_description: Temporarily fall back to the old Studio logged-in landing page. -# .. toggle_use_cases: temporary -# .. toggle_creation_date: 2025-03-14 -# .. toggle_target_removal_date: 2025-09-14 -# .. toggle_tickets: https://github.com/openedx/edx-platform/issues/36275 -# .. toggle_warning: In Ulmo, this toggle will be removed. Only the new (React-based) experience will be available. -LEGACY_STUDIO_HOME = WaffleFlag('legacy_studio.home', __name__) - - -def use_new_home_page(): - """ - Returns a boolean if new studio home page mfe is enabled - """ - return not LEGACY_STUDIO_HOME.is_enabled() - - # .. toggle_name: legacy_studio.custom_pages # .. toggle_implementation: WaffleFlag # .. toggle_default: False diff --git a/cms/djangoapps/contentstore/utils.py b/cms/djangoapps/contentstore/utils.py index efd6905745b3..04ed2787685a 100644 --- a/cms/djangoapps/contentstore/utils.py +++ b/cms/djangoapps/contentstore/utils.py @@ -15,7 +15,7 @@ from bs4 import BeautifulSoup from django.conf import settings -from django.core.exceptions import ObjectDoesNotExist, ValidationError +from django.core.exceptions import ImproperlyConfigured, ObjectDoesNotExist, ValidationError from django.urls import reverse from django.utils import translation from django.utils.text import Truncator @@ -50,7 +50,6 @@ use_new_files_uploads_page, use_new_grading_page, use_new_group_configurations_page, - use_new_home_page, use_new_import_page, use_new_schedule_details_page, use_new_textbooks_page, @@ -298,12 +297,15 @@ def get_studio_home_url(): """ Gets course authoring microfrontend URL for Studio Home view. """ - studio_home_url = None - if use_new_home_page(): - mfe_base_url = settings.COURSE_AUTHORING_MICROFRONTEND_URL - if mfe_base_url: - studio_home_url = f'{mfe_base_url}/home' - return studio_home_url + mfe_base_url = settings.COURSE_AUTHORING_MICROFRONTEND_URL + if mfe_base_url: + studio_home_url = f'{mfe_base_url}/home' + return studio_home_url + + raise ImproperlyConfigured( + "The COURSE_AUTHORING_MICROFRONTEND_URL must be configured. " + "Please set it to the base url for your authoring MFE." + ) def get_schedule_details_url(course_locator) -> str: diff --git a/cms/djangoapps/contentstore/views/course.py b/cms/djangoapps/contentstore/views/course.py index fa8769dc0cb9..3461d1b07779 100644 --- a/cms/djangoapps/contentstore/views/course.py +++ b/cms/djangoapps/contentstore/views/course.py @@ -86,7 +86,6 @@ from ..toggles import ( default_enable_flexible_peer_openassessments, use_new_course_outline_page, - use_new_home_page, use_new_updates_page, use_new_advanced_settings_page, use_new_grading_page, @@ -105,7 +104,6 @@ get_grading_url, get_group_configurations_context, get_group_configurations_url, - get_home_context, get_library_context, get_lms_link_for_item, get_proctored_exam_settings_url, @@ -652,11 +650,7 @@ def course_listing(request): """ List all courses and libraries available to the logged in user """ - if use_new_home_page(): - return redirect(get_studio_home_url()) - - home_context = get_home_context(request) - return render_to_response('index.html', home_context) + return redirect(get_studio_home_url()) @login_required diff --git a/cms/templates/widgets/header.html b/cms/templates/widgets/header.html index cc568299f40e..c5d75308eb6b 100644 --- a/cms/templates/widgets/header.html +++ b/cms/templates/widgets/header.html @@ -16,22 +16,12 @@
    From 0a400fbd1c54c48f0c0889eb8278382e10288893 Mon Sep 17 00:00:00 2001 From: Pradeep Date: Mon, 8 Dec 2025 17:52:26 +0530 Subject: [PATCH 139/351] fix: upgrade hls.js to version 1.6.15 and update related imports in video player tests (#57) * fix: upgrade hls.js to version 1.6.15 and update related imports in video player tests * fix: update hls import statements to use 'hls.js' for consistency across video modules * fix: update import statements for consistency in video player tests * fix: add eslint directive to suppress no-shadow-restricted-names warning in video player spec --- package-lock.json | 103 ++------------------- package.json | 2 +- xmodule/js/spec/video/video_player_spec.js | 8 +- xmodule/js/src/video/02_html5_hls_video.js | 16 ++-- xmodule/js/src/video/03_video_player.js | 4 +- 5 files changed, 25 insertions(+), 108 deletions(-) diff --git a/package-lock.json b/package-lock.json index 0da387590a61..6db7903427b4 100644 --- a/package-lock.json +++ b/package-lock.json @@ -39,7 +39,7 @@ "exports-loader": "0.6.4", "file-loader": "^6.2.0", "font-awesome": "4.7.0", - "hls.js": "0.14.17", + "hls.js": "^1.6.15", "imports-loader": "0.8.0", "jest-environment-jsdom": "^29.0.0", "jquery": "2.2.4", @@ -3556,15 +3556,12 @@ } }, "node_modules/@keyv/serialize": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/@keyv/serialize/-/serialize-1.0.3.tgz", - "integrity": "sha512-qnEovoOp5Np2JDGonIDL6Ayihw0RhnRh6vxPuHo4RDn1UOzwEo4AeIfpL6UGIrsceWrCMiVPgwRjbHu4vYFc3g==", + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/@keyv/serialize/-/serialize-1.1.1.tgz", + "integrity": "sha512-dXn3FZhPv0US+7dtJsIi2R+c7qWYiReoEh5zUntWCf4oSpMNib8FDhSoed6m3QyZdx5hK7iLFkYk3rNxwt8vTA==", "dev": true, "license": "MIT", - "peer": true, - "dependencies": { - "buffer": "^6.0.3" - } + "peer": true }, "node_modules/@nodelib/fs.scandir": { "version": "2.1.5", @@ -5668,28 +5665,6 @@ "node": ">= 0.6.0" } }, - "node_modules/base64-js": { - "version": "1.5.1", - "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", - "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==", - "dev": true, - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/feross" - }, - { - "type": "patreon", - "url": "https://www.patreon.com/feross" - }, - { - "type": "consulting", - "url": "https://feross.org/support" - } - ], - "license": "MIT", - "peer": true - }, "node_modules/base64id": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/base64id/-/base64id-1.0.0.tgz", @@ -5869,32 +5844,6 @@ "node-int64": "^0.4.0" } }, - "node_modules/buffer": { - "version": "6.0.3", - "resolved": "https://registry.npmjs.org/buffer/-/buffer-6.0.3.tgz", - "integrity": "sha512-FTiCpNxtwiZZHEZbcbTIcZjERVICn9yq/pDFkTl95/AxzD1naBctN7YO68riM/gLSDY7sdrMby8hofADYuuqOA==", - "dev": true, - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/feross" - }, - { - "type": "patreon", - "url": "https://www.patreon.com/feross" - }, - { - "type": "consulting", - "url": "https://feross.org/support" - } - ], - "license": "MIT", - "peer": true, - "dependencies": { - "base64-js": "^1.3.1", - "ieee754": "^1.2.1" - } - }, "node_modules/buffer-alloc": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/buffer-alloc/-/buffer-alloc-1.2.0.tgz", @@ -8703,6 +8652,7 @@ "version": "4.0.7", "resolved": "https://registry.npmjs.org/eventemitter3/-/eventemitter3-4.0.7.tgz", "integrity": "sha512-8guHBZCwKnFhYdHr2ysuRWErTwhoN2X8XELRlrRwpmfeY2jjuUN4taQMsULKUVo1K4DvZl+0pgfyoysHxvmvEw==", + "dev": true, "license": "MIT" }, "node_modules/events": { @@ -10094,14 +10044,10 @@ } }, "node_modules/hls.js": { - "version": "0.14.17", - "resolved": "https://registry.npmjs.org/hls.js/-/hls.js-0.14.17.tgz", - "integrity": "sha512-25A7+m6qqp6UVkuzUQ//VVh2EEOPYlOBg32ypr34bcPO7liBMOkKFvbjbCBfiPAOTA/7BSx1Dujft3Th57WyFg==", - "license": "Apache-2.0", - "dependencies": { - "eventemitter3": "^4.0.3", - "url-toolkit": "^2.1.6" - } + "version": "1.6.15", + "resolved": "https://registry.npmjs.org/hls.js/-/hls.js-1.6.15.tgz", + "integrity": "sha512-E3a5VwgXimGHwpRGV+WxRTKeSp2DW5DI5MWv34ulL3t5UNmyJWCQ1KmLEHbYzcfThfXG8amBL+fCYPneGHC4VA==", + "license": "Apache-2.0" }, "node_modules/hoist-non-react-statics": { "version": "3.3.2", @@ -10341,28 +10287,6 @@ "postcss": "^8.1.0" } }, - "node_modules/ieee754": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.2.1.tgz", - "integrity": "sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==", - "dev": true, - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/feross" - }, - { - "type": "patreon", - "url": "https://www.patreon.com/feross" - }, - { - "type": "consulting", - "url": "https://feross.org/support" - } - ], - "license": "BSD-3-Clause", - "peer": true - }, "node_modules/ignore": { "version": "3.3.10", "resolved": "https://registry.npmjs.org/ignore/-/ignore-3.3.10.tgz", @@ -15439,7 +15363,6 @@ "integrity": "sha512-ijt0LhxWClXBtc1RCt8H0WhlZLAdVX26nWbpsJy+Hblmp81d2F/pFsvlrJhJDDruFHM+ECtxP0H0HzGSrARkwg==", "deprecated": "You or someone you depend on is using Q, the JavaScript Promise library that gave JavaScript developers strong feelings about promises. They can almost certainly migrate to the native JavaScript promise now. Thank you literally everyone for joining me in this bet against the odds. Be excellent to each other.\n\n(For a CapTP with native promises, see @endo/eventual-send and @endo/captp)", "dev": true, - "license": "MIT", "engines": { "node": ">=0.6.0", "teleport": ">=0.2.0" @@ -20172,12 +20095,6 @@ "requires-port": "^1.0.0" } }, - "node_modules/url-toolkit": { - "version": "2.2.5", - "resolved": "https://registry.npmjs.org/url-toolkit/-/url-toolkit-2.2.5.tgz", - "integrity": "sha512-mtN6xk+Nac+oyJ/PrI7tzfmomRVNFIWKUbG8jdYFt52hxbiReFAXIjYskvu64/dvuW71IcB7lV8l0HvZMac6Jg==", - "license": "Apache-2.0" - }, "node_modules/use": { "version": "3.1.1", "resolved": "https://registry.npmjs.org/use/-/use-3.1.1.tgz", diff --git a/package.json b/package.json index a8b762450e44..a3dfdd45feff 100644 --- a/package.json +++ b/package.json @@ -64,7 +64,7 @@ "exports-loader": "0.6.4", "file-loader": "^6.2.0", "font-awesome": "4.7.0", - "hls.js": "0.14.17", + "hls.js": "^1.6.15", "imports-loader": "0.8.0", "jest-environment-jsdom": "^29.0.0", "jquery": "2.2.4", diff --git a/xmodule/js/spec/video/video_player_spec.js b/xmodule/js/spec/video/video_player_spec.js index 4845fc80fa1d..5782d760d830 100644 --- a/xmodule/js/spec/video/video_player_spec.js +++ b/xmodule/js/spec/video/video_player_spec.js @@ -1,12 +1,12 @@ /* global YT */ // eslint-disable-next-line no-shadow-restricted-names -(function(require, define, undefined) { +(function() { 'use strict'; require( ['video/03_video_player.js', 'hls', 'underscore'], - function(VideoPlayer, HLS, _) { + function(VideoPlayer, Hls, _) { describe('VideoPlayer', function() { var STATUS = window.STATUS, state, @@ -982,7 +982,7 @@ describe('on safari', function() { beforeEach(function() { - spyOn(HLS, 'isSupported').and.returnValue(false); + spyOn(Hls, 'isSupported').and.returnValue(false); state = jasmine.initializeHLSPlayer(); state.canPlayHLS = true; state.browserIsSafari = true; @@ -996,7 +996,7 @@ describe('HLS Video Errors', function() { beforeEach(function() { - spyOn(HLS, 'isSupported').and.returnValue(false); + spyOn(Hls, 'isSupported').and.returnValue(false); state = jasmine.initializeHLSPlayer({sources: ['/base/fixtures/hls/hls.m3u8']}); }); diff --git a/xmodule/js/src/video/02_html5_hls_video.js b/xmodule/js/src/video/02_html5_hls_video.js index cb6a1a2fda27..00cc967731f8 100644 --- a/xmodule/js/src/video/02_html5_hls_video.js +++ b/xmodule/js/src/video/02_html5_hls_video.js @@ -8,7 +8,7 @@ 'use strict'; define('video/02_html5_hls_video.js', ['underscore', 'video/02_html5_video.js', 'hls'], - function(_, HTML5Video, HLS) { + function(_, HTML5Video, Hls) { var HLSVideo = {}; HLSVideo.Player = (function() { @@ -45,16 +45,16 @@ } else { // load auto start if auto_advance is enabled if (config.state.auto_advance) { - this.hls = new HLS({autoStartLoad: true}); + this.hls = new Hls({autoStartLoad: true}); } else { - this.hls = new HLS({autoStartLoad: false}); + this.hls = new Hls({autoStartLoad: false}); } this.hls.loadSource(config.videoSources[0]); this.hls.attachMedia(this.video); - this.hls.on(HLS.Events.ERROR, this.onError.bind(this)); + this.hls.on(Hls.Events.ERROR, this.onError.bind(this)); - this.hls.on(HLS.Events.MANIFEST_PARSED, function(event, data) { + this.hls.on(Hls.Events.MANIFEST_PARSED, function(event, data) { console.log( '[HLS Video]: MANIFEST_PARSED, qualityLevelsInfo: ', data.levels.map(function(level) { @@ -66,7 +66,7 @@ ); self.config.onReadyHLS(); }); - this.hls.on(HLS.Events.LEVEL_SWITCHED, function(event, data) { + this.hls.on(Hls.Events.LEVEL_SWITCHED, function(event, data) { var level = self.hls.levels[data.level]; console.log( '[HLS Video]: LEVEL_SWITCHED, qualityLevelInfo: ', @@ -114,14 +114,14 @@ Player.prototype.onError = function(event, data) { if (data.fatal) { switch (data.type) { - case HLS.ErrorTypes.NETWORK_ERROR: + case Hls.ErrorTypes.NETWORK_ERROR: console.error( '[HLS Video]: Fatal network error encountered, try to recover. Details: %s', data.details ); this.hls.startLoad(); break; - case HLS.ErrorTypes.MEDIA_ERROR: + case Hls.ErrorTypes.MEDIA_ERROR: console.error( '[HLS Video]: Fatal media error encountered, try to recover. Details: %s', data.details diff --git a/xmodule/js/src/video/03_video_player.js b/xmodule/js/src/video/03_video_player.js index 6215d54689ad..d682a6ccfac7 100644 --- a/xmodule/js/src/video/03_video_player.js +++ b/xmodule/js/src/video/03_video_player.js @@ -4,7 +4,7 @@ define( 'video/03_video_player.js', ['video/02_html5_video.js', 'video/02_html5_hls_video.js', 'video/00_resizer.js', 'hls', 'underscore', '../time.js'], - function(HTML5Video, HTML5HLSVideo, Resizer, HLS, _, Time) { + function(HTML5Video, HTML5HLSVideo, Resizer, Hls, _, Time) { var dfd = $.Deferred(), VideoPlayer = function(state) { state.videoPlayer = {}; @@ -157,7 +157,7 @@ // Browser can play HLS videos if either `Media Source Extensions` // feature is supported or browser is safari (native HLS support) - state.canPlayHLS = state.HLSVideoSources.length > 0 && (HLS.isSupported() || state.browserIsSafari); + state.canPlayHLS = state.HLSVideoSources.length > 0 && (Hls.isSupported() || state.browserIsSafari); state.HLSOnlySources = state.config.sources.length > 0 && state.config.sources.length === state.HLSVideoSources.length; From 0ee2faa6ea1208e0988ffe482ab1c7d456eccb61 Mon Sep 17 00:00:00 2001 From: Ahtisham Shahid Date: Tue, 9 Dec 2025 10:20:37 +0500 Subject: [PATCH 140/351] chore: updated pref settings for misc notification types (#37738) --- openedx/core/djangoapps/notifications/base_notification.py | 4 ++-- openedx/core/djangoapps/notifications/tests/test_views.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/openedx/core/djangoapps/notifications/base_notification.py b/openedx/core/djangoapps/notifications/base_notification.py index 5264053ace5a..4b9393830eb9 100644 --- a/openedx/core/djangoapps/notifications/base_notification.py +++ b/openedx/core/djangoapps/notifications/base_notification.py @@ -178,7 +178,7 @@ 'is_core': False, 'info': '', 'web': True, - 'email': False, + 'email': True, 'push': False, 'email_cadence': EmailCadence.DAILY, 'non_editable': ['push'], @@ -236,7 +236,7 @@ 'is_core': False, 'info': '', 'web': True, - 'email': False, + 'email': True, 'email_cadence': EmailCadence.DAILY, 'push': False, 'non_editable': ['push'], diff --git a/openedx/core/djangoapps/notifications/tests/test_views.py b/openedx/core/djangoapps/notifications/tests/test_views.py index 7148295dcf50..5e09a86c5914 100644 --- a/openedx/core/djangoapps/notifications/tests/test_views.py +++ b/openedx/core/djangoapps/notifications/tests/test_views.py @@ -604,7 +604,7 @@ def setUp(self): }, "new_instructor_all_learners_post": { "web": True, - "email": False, + "email": True, "push": False, "email_cadence": "Daily" }, @@ -628,7 +628,7 @@ def setUp(self): "notification_types": { "course_updates": { "web": True, - "email": False, + "email": True, "push": False, "email_cadence": "Daily" }, From 70330921c1854b78a6d3bcf4420b7355174f0a04 Mon Sep 17 00:00:00 2001 From: Krish Tyagi Date: Tue, 9 Dec 2025 16:01:28 +0530 Subject: [PATCH 141/351] fix: update geolite workflow (#40) * fix: update geolite workflow --- .github/workflows/update-geolite-database.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/update-geolite-database.yml b/.github/workflows/update-geolite-database.yml index 484fa167a371..c13cd5469722 100644 --- a/.github/workflows/update-geolite-database.yml +++ b/.github/workflows/update-geolite-database.yml @@ -8,7 +8,7 @@ on: branch: description: "Target branch against which to create PR" required: false - default: "master" + default: "release-ulmo" env: MAXMIND_URL: "https://download.maxmind.com/app/geoip_download?edition_id=GeoLite2-Country&license_key=${{ secrets.MAXMIND_LICENSE_KEY }}&suffix=tar.gz" @@ -79,13 +79,13 @@ jobs: --title "Update GeoLite Database" \ --body "PR generated by workflow `${{ github.workflow }}` on behalf of @${{ github.actor }}." \ --head $BRANCH \ - --base 'master' \ - --reviewer 'feanil' \ + --base 'release-ulmo' \ + --reviewer 'edx/orbi-bom' \ | grep -o 'https://github.com/.*/pull/[0-9]*') echo "PR Created: ${PR_URL}" echo "pull-request-url=$PR_URL" >> $GITHUB_OUTPUT env: - GH_TOKEN: ${{ github.token }} + GH_TOKEN: ${{ secrets.GH_PAT_WITH_ORG }} - name: Job summary run: | From 9956dd70e79cde6c63615369f858b6caf4c136c7 Mon Sep 17 00:00:00 2001 From: Asespinel <79876430+Asespinel@users.noreply.github.com> Date: Wed, 3 Dec 2025 11:28:49 -0500 Subject: [PATCH 142/351] fix: Course search pill not cleared when text deleted. (#37709) * fix: Course search pill not cleared when text deleted * chore: fix spacing Co-authored-by: Feanil Patel --------- Co-authored-by: Feanil Patel --- lms/static/js/discovery/discovery_factory.js | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/lms/static/js/discovery/discovery_factory.js b/lms/static/js/discovery/discovery_factory.js index d26841f86fca..0e44d8154a84 100644 --- a/lms/static/js/discovery/discovery_factory.js +++ b/lms/static/js/discovery/discovery_factory.js @@ -30,8 +30,11 @@ } listing = new CoursesListing({model: courseListingModel}); - dispatcher.listenTo(form, 'search', function(query) { + dispatcher.listenTo(form, "search", function (query) { form.showLoadingIndicator(); + if (!query || query.trim() === "") { + filters.remove("search_query"); + } search.performSearch(query, filters.getTerms()); }); From f3b971913dd635fa46b86e52f10f52cec9030035 Mon Sep 17 00:00:00 2001 From: Asad Ali Date: Thu, 20 Nov 2025 22:05:14 +0500 Subject: [PATCH 143/351] revert: feat: [FC-0092] Optimize Course Info Blocks API (#37122) (#37661) This reverts commit 7cd4170ca7bfd29612fe4b8b469df539d64e327f. --- lms/djangoapps/course_api/blocks/api.py | 24 +------ lms/djangoapps/course_api/blocks/utils.py | 20 ------ lms/djangoapps/course_api/blocks/views.py | 43 ----------- lms/djangoapps/courseware/courses.py | 6 +- lms/djangoapps/grades/course_data.py | 8 +-- .../tests/test_course_info_views.py | 72 ------------------- 6 files changed, 5 insertions(+), 168 deletions(-) diff --git a/lms/djangoapps/course_api/blocks/api.py b/lms/djangoapps/course_api/blocks/api.py index 80c2bb7fc5ff..a79e9759e191 100644 --- a/lms/djangoapps/course_api/blocks/api.py +++ b/lms/djangoapps/course_api/blocks/api.py @@ -1,7 +1,7 @@ """ API function for retrieving course blocks data """ -from edx_django_utils.cache import RequestCache + import lms.djangoapps.course_blocks.api as course_blocks_api from lms.djangoapps.course_blocks.transformers.access_denied_filter import AccessDeniedMessageFilterTransformer @@ -14,7 +14,6 @@ from .toggles import HIDE_ACCESS_DENIALS_FLAG from .transformers.blocks_api import BlocksAPITransformer from .transformers.milestones import MilestonesAndSpecialExamsTransformer -from .utils import COURSE_API_REQUEST_CACHE_NAMESPACE, REUSABLE_BLOCKS_CACHE_KEY def get_blocks( @@ -30,7 +29,6 @@ def get_blocks( block_types_filter=None, hide_access_denials=False, allow_start_dates_in_future=False, - cache_with_future_dates=False, ): """ Return a serialized representation of the course blocks. @@ -63,7 +61,6 @@ def get_blocks( allow_start_dates_in_future (bool): When True, will allow blocks to be returned that can bypass the StartDateTransformer's filter to show blocks with start dates in the future. - cache_with_future_dates (bool): When True, will use the block caching logic using RequestCache """ if HIDE_ACCESS_DENIALS_FLAG.is_enabled(): @@ -121,10 +118,6 @@ def get_blocks( ), ] - if cache_with_future_dates: - # Include future dates such that get_course_assignments can reuse the block structure from RequestCache - allow_start_dates_in_future = True - # transform blocks = course_blocks_api.get_course_blocks( user, @@ -135,19 +128,6 @@ def get_blocks( include_has_scheduled_content=include_has_scheduled_content ) - if cache_with_future_dates: - # Store a copy of the transformed, but still unfiltered, course blocks in RequestCache to be reused - # wherever possible for optimization. Copying is required to make sure the cached structure is not mutated - # by the filtering below. - request_cache = RequestCache(COURSE_API_REQUEST_CACHE_NAMESPACE) - request_cache.set(REUSABLE_BLOCKS_CACHE_KEY, blocks.copy()) - - # Since we included blocks with future start dates in our block structure, - # we need to include the 'start' field to filter out such blocks before returning the response. - # If 'start' field is not requested, it will be removed from the response. - requested_fields = set(requested_fields) - requested_fields.add('start') - # filter blocks by types if block_types_filter: block_keys_to_remove = [] @@ -162,7 +142,7 @@ def get_blocks( serializer_context = { 'request': request, 'block_structure': blocks, - 'requested_fields': requested_fields, + 'requested_fields': requested_fields or [], } if return_type == 'dict': diff --git a/lms/djangoapps/course_api/blocks/utils.py b/lms/djangoapps/course_api/blocks/utils.py index 0686abc2fac1..6f371624b7df 100644 --- a/lms/djangoapps/course_api/blocks/utils.py +++ b/lms/djangoapps/course_api/blocks/utils.py @@ -1,7 +1,6 @@ """ Utils for Blocks """ -from edx_django_utils.cache import RequestCache from rest_framework.utils.serializer_helpers import ReturnList from openedx.core.djangoapps.discussions.models import ( @@ -10,10 +9,6 @@ ) -COURSE_API_REQUEST_CACHE_NAMESPACE = "course_api" -REUSABLE_BLOCKS_CACHE_KEY = "reusable_transformed_blocks" - - def filter_discussion_xblocks_from_response(response, course_key): """ Removes discussion xblocks if discussion provider is openedx. @@ -68,18 +63,3 @@ def filter_discussion_xblocks_from_response(response, course_key): response.data['blocks'] = filtered_blocks return response - - -def get_cached_transformed_blocks(): - """ - Helper function to get an unfiltered course structure from RequestCache, - including blocks with start dates in the future. - - Caution: For performance reasons, the function returns the structure object itself, not its copy. - This means the retrieved structure is supposed to be read-only and should not be mutated by consumers. - """ - request_cache = RequestCache(COURSE_API_REQUEST_CACHE_NAMESPACE) - cached_response = request_cache.get_cached_response(REUSABLE_BLOCKS_CACHE_KEY) - reusable_transformed_blocks = cached_response.value if cached_response.is_found else None - - return reusable_transformed_blocks diff --git a/lms/djangoapps/course_api/blocks/views.py b/lms/djangoapps/course_api/blocks/views.py index 7f81861b9b7b..96679a562957 100644 --- a/lms/djangoapps/course_api/blocks/views.py +++ b/lms/djangoapps/course_api/blocks/views.py @@ -2,7 +2,6 @@ CourseBlocks API views """ -from datetime import datetime, timezone from django.core.exceptions import ValidationError from django.db import transaction @@ -238,7 +237,6 @@ def list(self, request, usage_key_string, hide_access_denials=False): # pylint: params.cleaned_data['return_type'], params.cleaned_data.get('block_types_filter', None), hide_access_denials=hide_access_denials, - cache_with_future_dates=True ) ) # If the username is an empty string, and not None, then we are requesting @@ -341,50 +339,9 @@ def list(self, request, hide_access_denials=False): # pylint: disable=arguments if not root: raise ValidationError(f"Unable to find course block in '{course_key_string}'") - # Earlier we included blocks with future start dates in the collected/cached block structure. - # Now we need to emulate allow_start_dates_in_future=False by removing any such blocks. - include_start = "start" in request.query_params['requested_fields'] - self.remove_future_blocks(course_blocks, include_start) - recurse_mark_complete(root, course_blocks) return response - @staticmethod - def remove_future_blocks(course_blocks, include_start: bool): - """ - Mutates course_blocks in place: - - removes blocks whose 'start' is in the future - - also removes references to them from parents' 'children' lists - - removes 'start' key from all blocks if it wasn't requested - """ - if not course_blocks: - return course_blocks - - now = datetime.now(timezone.utc) - - # 1. Collect IDs of blocks to remove - to_remove = set() - for block_id, block in course_blocks.items(): - get_field = block.get if include_start else block.pop - start = get_field("start") - if start and start > now: - to_remove.add(block_id) - - if not to_remove: - return course_blocks - - # 2. Remove the blocks themselves - for block_id in to_remove: - course_blocks.pop(block_id, None) - - # 3. Clean up children lists - for block in course_blocks.values(): - children = block.get("children") - if children: - block["children"] = [cid for cid in children if cid not in to_remove] - - return course_blocks - @method_decorator(transaction.non_atomic_requests, name='dispatch') @view_auth_classes(is_authenticated=False) diff --git a/lms/djangoapps/courseware/courses.py b/lms/djangoapps/courseware/courses.py index 3b527cda4dfa..bbf9d5394705 100644 --- a/lms/djangoapps/courseware/courses.py +++ b/lms/djangoapps/courseware/courses.py @@ -26,7 +26,6 @@ from common.djangoapps.static_replace import replace_static_urls from common.djangoapps.util.date_utils import strftime_localized from lms.djangoapps import branding -from lms.djangoapps.course_api.blocks.utils import get_cached_transformed_blocks from lms.djangoapps.course_blocks.api import get_course_blocks from lms.djangoapps.courseware.access import has_access from lms.djangoapps.courseware.access_response import ( @@ -637,10 +636,7 @@ def get_course_assignments(course_key, user, include_access=False, include_witho store = modulestore() course_usage_key = store.make_course_usage_key(course_key) - - block_data = get_cached_transformed_blocks() or get_course_blocks( - user, course_usage_key, allow_start_dates_in_future=True, include_completion=True - ) + block_data = get_course_blocks(user, course_usage_key, allow_start_dates_in_future=True, include_completion=True) now = datetime.now(pytz.UTC) assignments = [] diff --git a/lms/djangoapps/grades/course_data.py b/lms/djangoapps/grades/course_data.py index 523d6e6df38d..5464c4f88105 100644 --- a/lms/djangoapps/grades/course_data.py +++ b/lms/djangoapps/grades/course_data.py @@ -2,12 +2,12 @@ Code used to get and cache the requested course-data """ + from lms.djangoapps.course_blocks.api import get_course_blocks from openedx.core.djangoapps.content.block_structure.api import get_block_structure_manager from xmodule.modulestore.django import modulestore # lint-amnesty, pylint: disable=wrong-import-order from .transformer import GradesTransformer -from ..course_api.blocks.utils import get_cached_transformed_blocks class CourseData: @@ -56,11 +56,7 @@ def location(self): # lint-amnesty, pylint: disable=missing-function-docstring @property def structure(self): # lint-amnesty, pylint: disable=missing-function-docstring if self._structure is None: - # The get_course_blocks function proved to be a major time sink during a request at "blocks/". - # This caching logic helps improve the response time by getting a copy of the already transformed, but still - # unfiltered, course blocks from RequestCache and thus reducing the number of times that - # the get_course_blocks function is called. - self._structure = get_cached_transformed_blocks() or get_course_blocks( + self._structure = get_course_blocks( self.user, self.location, collected_block_structure=self._collected_block_structure, diff --git a/lms/djangoapps/mobile_api/tests/test_course_info_views.py b/lms/djangoapps/mobile_api/tests/test_course_info_views.py index b7cdc6b03961..efb3f7d9fdbb 100644 --- a/lms/djangoapps/mobile_api/tests/test_course_info_views.py +++ b/lms/djangoapps/mobile_api/tests/test_course_info_views.py @@ -432,78 +432,6 @@ def test_extend_sequential_info_with_assignment_progress_for_other_types(self, b for block_info in response.data['blocks'].values(): self.assertNotEqual('assignment_progress', block_info) - def test_response_keys(self): - response = self.verify_response(url=self.url) - data = response.data - - expected_top_level_keys = { - 'blocks', - 'certificate', - 'course_about', - 'course_access_details', - 'course_handouts', - 'course_modes', - 'course_progress', - 'course_sharing_utm_parameters', - 'course_updates', - 'deprecate_youtube', - 'discussion_url', - 'end', - 'enrollment_details', - 'id', - 'is_self_paced', - 'media', - 'name', - 'number', - 'org', - 'org_logo', - 'root', - 'start', - 'start_display', - 'start_type' - } - expected_course_access_keys = { - "has_unmet_prerequisites", - "is_too_early", - "is_staff", - "audit_access_expires", - "courseware_access" - } - expected_courseware_access_keys = { - "has_access", - "error_code", - "developer_message", - "user_message", - "additional_context_user_message", - "user_fragment" - } - expected_enrollment_details_keys = {"created", "mode", "is_active", "upgrade_deadline"} - expected_media_keys = {"image"} - expected_image_keys = {"raw", "small", "large"} - expected_course_sharing_keys = {"facebook", "twitter"} - expected_course_modes_keys = {"slug", "sku", "android_sku", "ios_sku", "min_price"} - expected_course_progress_keys = {"total_assignments_count", "assignments_completed"} - - self.assertSetEqual(set(data), expected_top_level_keys) - self.assertSetEqual(set(data["course_access_details"]), expected_course_access_keys) - self.assertSetEqual(set(data["course_access_details"]["courseware_access"]), expected_courseware_access_keys) - self.assertSetEqual(set(data["enrollment_details"]), expected_enrollment_details_keys) - self.assertSetEqual(set(data["media"]), expected_media_keys) - self.assertSetEqual(set(data["media"]["image"]), expected_image_keys) - self.assertSetEqual(set(data["course_sharing_utm_parameters"]), expected_course_sharing_keys) - self.assertSetEqual(set(data["course_modes"][0]), expected_course_modes_keys) - self.assertSetEqual(set(data["course_progress"]), expected_course_progress_keys) - - def test_block_count_depends_on_depth_in_request_params(self): - response_depth_zero = self.verify_response(url=self.url, params={'depth': 0}) - response_depth_one = self.verify_response(url=self.url, params={'depth': 1}) - blocks_depth_zero = [block for block in self.store.get_items(self.course_key) if block.category == "course"] - blocks_depth_one = [ - block for block in self.store.get_items(self.course_key) if block.category in ("course", "chapter") - ] - self.assertEqual(len(response_depth_zero.data["blocks"]), len(blocks_depth_zero)) - self.assertEqual(len(response_depth_one.data["blocks"]), len(blocks_depth_one)) - class TestCourseEnrollmentDetailsView(MobileAPITestCase, MilestonesTestCaseMixin): # lint-amnesty, pylint: disable=test-inherits-tests """ From ba5113c446027a03db25ea82c504d72293f72962 Mon Sep 17 00:00:00 2001 From: David Ormsbee Date: Wed, 3 Dec 2025 16:49:08 -0500 Subject: [PATCH 144/351] fix: don't send emails on library backup/restore There is no way to resume either the backup or restore library actions, i.e. if you navigate away from it, you have to do it again. This is a limitation of the current UI because we wanted to get something quick and simple in for Ulmo, but it also reflects the fact that library backup/restore should be much faster than course import/export has historically been. In any case, sending an email for a 5-10 second task is unnecessary and distracting, so this commit suppresses the email. Note: I'm using local imports to get around the fact that the content_libraries public API is used by content_libraries/tasks.py which defines the tasks. I can't import from content_libraries/tasks.py directly, because that would violate import linter rules forbidding other apps from importing things outside of api.py. This isn't ideal, but it keeps the fix small and it keeps the logic in the content_libraries app. --- cms/djangoapps/cms_user_tasks/signals.py | 25 ++++++++++++++++++- .../content_libraries/api/libraries.py | 14 +++++++++++ .../djangoapps/content_libraries/tasks.py | 7 ++++-- .../content_libraries/tests/test_tasks.py | 10 ++++++-- 4 files changed, 51 insertions(+), 5 deletions(-) diff --git a/cms/djangoapps/cms_user_tasks/signals.py b/cms/djangoapps/cms_user_tasks/signals.py index 40bfd5781825..e6ddba747d5f 100644 --- a/cms/djangoapps/cms_user_tasks/signals.py +++ b/cms/djangoapps/cms_user_tasks/signals.py @@ -12,6 +12,7 @@ from cms.djangoapps.contentstore.toggles import bypass_olx_failure_enabled from cms.djangoapps.contentstore.utils import course_import_olx_validation_is_enabled +from openedx.core.djangoapps.content_libraries.api import is_library_backup_task, is_library_restore_task from .tasks import send_task_complete_email @@ -64,6 +65,28 @@ def get_olx_validation_from_artifact(): if olx_artifact and not bypass_olx_failure_enabled(): return olx_artifact.text + def should_skip_end_of_task_email(task_name) -> bool: + """ + Studio tasks generally send an email when finished, but not always. + + Some tasks can last many minutes, e.g. course import/export. For these + tasks, there is a high chance that the user has navigated away and will + want to check back in later. Yet email notification is unnecessary and + distracting for things like the Library restore task, which is + relatively quick and cannot be resumed (i.e. if you navigate away, you + have to upload again). + + The task_name passed in will be lowercase. + """ + # We currently have to pattern match on the name to differentiate + # between tasks. A better long term solution would be to add a separate + # task type identifier field to Django User Tasks. + return ( + is_library_content_update(task_name) or + is_library_backup_task(task_name) or + is_library_restore_task(task_name) + ) + status = kwargs['status'] # Only send email when the entire task is complete, should only send when @@ -72,7 +95,7 @@ def get_olx_validation_from_artifact(): task_name = status.name.lower() # Also suppress emails on library content XBlock updates (too much like spam) - if is_library_content_update(task_name): + if should_skip_end_of_task_email(task_name): LOGGER.info(f"Suppressing end-of-task email on task {task_name}") return diff --git a/openedx/core/djangoapps/content_libraries/api/libraries.py b/openedx/core/djangoapps/content_libraries/api/libraries.py index 66281addb643..d77061d583be 100644 --- a/openedx/core/djangoapps/content_libraries/api/libraries.py +++ b/openedx/core/djangoapps/content_libraries/api/libraries.py @@ -108,6 +108,8 @@ "get_backup_task_status", "assign_library_role_to_user", "user_has_permission_across_lib_authz_systems", + "is_library_backup_task", + "is_library_restore_task", ] @@ -851,3 +853,15 @@ def _is_legacy_permission(permission: str) -> bool: or the new openedx-authz system. """ return permission in LEGACY_LIB_PERMISSIONS + + +def is_library_backup_task(task_name: str) -> bool: + """Case-insensitive match to see if a task is a library backup.""" + from ..tasks import LibraryBackupTask # avoid circular import error + return task_name.startswith(LibraryBackupTask.NAME_PREFIX.lower()) + + +def is_library_restore_task(task_name: str) -> bool: + """Case-insensitive match to see if a task is a library restore.""" + from ..tasks import LibraryRestoreTask # avoid circular import error + return task_name.startswith(LibraryRestoreTask.NAME_PREFIX.lower()) diff --git a/openedx/core/djangoapps/content_libraries/tasks.py b/openedx/core/djangoapps/content_libraries/tasks.py index c54779a8e621..bbbf847bfbc7 100644 --- a/openedx/core/djangoapps/content_libraries/tasks.py +++ b/openedx/core/djangoapps/content_libraries/tasks.py @@ -512,6 +512,7 @@ class LibraryBackupTask(UserTask): # pylint: disable=abstract-method """ Base class for tasks related with Library backup functionality. """ + NAME_PREFIX = "Library Learning Package Backup" @classmethod def generate_name(cls, arguments_dict) -> str: @@ -529,7 +530,7 @@ def generate_name(cls, arguments_dict) -> str: str: The generated name """ key = arguments_dict['library_key_str'] - return f'Backup of {key}' + return f'{cls.NAME_PREFIX} of {key}' @shared_task(base=LibraryBackupTask, bind=True) @@ -591,10 +592,12 @@ class LibraryRestoreTask(UserTask): ERROR_LOG_ARTIFACT_NAME = 'Error log' + NAME_PREFIX = "Library Learning Package Restore" + @classmethod def generate_name(cls, arguments_dict): storage_path = arguments_dict['storage_path'] - return f'learning package restore of {storage_path}' + return f'{cls.NAME_PREFIX} of {storage_path}' def fail_with_error_log(self, logfile) -> None: """ diff --git a/openedx/core/djangoapps/content_libraries/tests/test_tasks.py b/openedx/core/djangoapps/content_libraries/tests/test_tasks.py index ac64bc86ef28..a200471c00bd 100644 --- a/openedx/core/djangoapps/content_libraries/tests/test_tasks.py +++ b/openedx/core/djangoapps/content_libraries/tests/test_tasks.py @@ -1,6 +1,7 @@ """ Unit tests for content libraries Celery tasks """ +from unittest import mock from django.test import override_settings from ..models import ContentLibrary @@ -14,6 +15,7 @@ class ContentLibraryBackupTaskTest(ContentLibrariesRestApiTest): """ Tests for Content Library export task. """ + SEND_TASK_COMPLETE_FN = 'cms.djangoapps.cms_user_tasks.tasks.send_task_complete_email.delay' def setUp(self) -> None: super().setUp() @@ -31,7 +33,9 @@ def test_backup_task_returns_task_id(self): @override_settings(CMS_BASE="test.com") def test_backup_task_success(self): - result = backup_library.delay(self.user.id, str(self.lib1.library_key)) + with mock.patch(self.SEND_TASK_COMPLETE_FN) as send_task_complete_email: + result = backup_library.delay(self.user.id, str(self.lib1.library_key)) + send_task_complete_email.assert_not_called() assert result.state == 'SUCCESS' # Ensure an artifact was created with the output file artifact = UserTaskArtifact.objects.filter(status__task_id=result.task_id, name='Output').first() @@ -44,7 +48,9 @@ def test_backup_task_success(self): assert b'origin_server = "test.com"' in content def test_backup_task_failure(self): - result = backup_library.delay(self.user.id, self.wrong_task_id) + with mock.patch(self.SEND_TASK_COMPLETE_FN) as send_task_complete_email: + result = backup_library.delay(self.user.id, self.wrong_task_id) + send_task_complete_email.assert_not_called() assert result.state == 'FAILURE' # Ensure an error artifact was created artifact = UserTaskArtifact.objects.filter(status__task_id=result.task_id, name='Error').first() From f348776bd6440a381216eb54831f8ad001cba95e Mon Sep 17 00:00:00 2001 From: Pradeep Date: Thu, 11 Dec 2025 14:48:19 +0530 Subject: [PATCH 145/351] =?UTF-8?q?Revert=20"fix:=20upgrade=20hls.js=20to?= =?UTF-8?q?=20version=201.6.15=20and=20update=20related=20imports=20in=20v?= =?UTF-8?q?=E2=80=A6"=20(#59)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This reverts commit 0a400fbd1c54c48f0c0889eb8278382e10288893. --- package-lock.json | 103 +++++++++++++++++++-- package.json | 2 +- xmodule/js/spec/video/video_player_spec.js | 8 +- xmodule/js/src/video/02_html5_hls_video.js | 16 ++-- xmodule/js/src/video/03_video_player.js | 4 +- 5 files changed, 108 insertions(+), 25 deletions(-) diff --git a/package-lock.json b/package-lock.json index 6db7903427b4..0da387590a61 100644 --- a/package-lock.json +++ b/package-lock.json @@ -39,7 +39,7 @@ "exports-loader": "0.6.4", "file-loader": "^6.2.0", "font-awesome": "4.7.0", - "hls.js": "^1.6.15", + "hls.js": "0.14.17", "imports-loader": "0.8.0", "jest-environment-jsdom": "^29.0.0", "jquery": "2.2.4", @@ -3556,12 +3556,15 @@ } }, "node_modules/@keyv/serialize": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/@keyv/serialize/-/serialize-1.1.1.tgz", - "integrity": "sha512-dXn3FZhPv0US+7dtJsIi2R+c7qWYiReoEh5zUntWCf4oSpMNib8FDhSoed6m3QyZdx5hK7iLFkYk3rNxwt8vTA==", + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/@keyv/serialize/-/serialize-1.0.3.tgz", + "integrity": "sha512-qnEovoOp5Np2JDGonIDL6Ayihw0RhnRh6vxPuHo4RDn1UOzwEo4AeIfpL6UGIrsceWrCMiVPgwRjbHu4vYFc3g==", "dev": true, "license": "MIT", - "peer": true + "peer": true, + "dependencies": { + "buffer": "^6.0.3" + } }, "node_modules/@nodelib/fs.scandir": { "version": "2.1.5", @@ -5665,6 +5668,28 @@ "node": ">= 0.6.0" } }, + "node_modules/base64-js": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", + "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT", + "peer": true + }, "node_modules/base64id": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/base64id/-/base64id-1.0.0.tgz", @@ -5844,6 +5869,32 @@ "node-int64": "^0.4.0" } }, + "node_modules/buffer": { + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/buffer/-/buffer-6.0.3.tgz", + "integrity": "sha512-FTiCpNxtwiZZHEZbcbTIcZjERVICn9yq/pDFkTl95/AxzD1naBctN7YO68riM/gLSDY7sdrMby8hofADYuuqOA==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT", + "peer": true, + "dependencies": { + "base64-js": "^1.3.1", + "ieee754": "^1.2.1" + } + }, "node_modules/buffer-alloc": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/buffer-alloc/-/buffer-alloc-1.2.0.tgz", @@ -8652,7 +8703,6 @@ "version": "4.0.7", "resolved": "https://registry.npmjs.org/eventemitter3/-/eventemitter3-4.0.7.tgz", "integrity": "sha512-8guHBZCwKnFhYdHr2ysuRWErTwhoN2X8XELRlrRwpmfeY2jjuUN4taQMsULKUVo1K4DvZl+0pgfyoysHxvmvEw==", - "dev": true, "license": "MIT" }, "node_modules/events": { @@ -10044,10 +10094,14 @@ } }, "node_modules/hls.js": { - "version": "1.6.15", - "resolved": "https://registry.npmjs.org/hls.js/-/hls.js-1.6.15.tgz", - "integrity": "sha512-E3a5VwgXimGHwpRGV+WxRTKeSp2DW5DI5MWv34ulL3t5UNmyJWCQ1KmLEHbYzcfThfXG8amBL+fCYPneGHC4VA==", - "license": "Apache-2.0" + "version": "0.14.17", + "resolved": "https://registry.npmjs.org/hls.js/-/hls.js-0.14.17.tgz", + "integrity": "sha512-25A7+m6qqp6UVkuzUQ//VVh2EEOPYlOBg32ypr34bcPO7liBMOkKFvbjbCBfiPAOTA/7BSx1Dujft3Th57WyFg==", + "license": "Apache-2.0", + "dependencies": { + "eventemitter3": "^4.0.3", + "url-toolkit": "^2.1.6" + } }, "node_modules/hoist-non-react-statics": { "version": "3.3.2", @@ -10287,6 +10341,28 @@ "postcss": "^8.1.0" } }, + "node_modules/ieee754": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.2.1.tgz", + "integrity": "sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "BSD-3-Clause", + "peer": true + }, "node_modules/ignore": { "version": "3.3.10", "resolved": "https://registry.npmjs.org/ignore/-/ignore-3.3.10.tgz", @@ -15363,6 +15439,7 @@ "integrity": "sha512-ijt0LhxWClXBtc1RCt8H0WhlZLAdVX26nWbpsJy+Hblmp81d2F/pFsvlrJhJDDruFHM+ECtxP0H0HzGSrARkwg==", "deprecated": "You or someone you depend on is using Q, the JavaScript Promise library that gave JavaScript developers strong feelings about promises. They can almost certainly migrate to the native JavaScript promise now. Thank you literally everyone for joining me in this bet against the odds. Be excellent to each other.\n\n(For a CapTP with native promises, see @endo/eventual-send and @endo/captp)", "dev": true, + "license": "MIT", "engines": { "node": ">=0.6.0", "teleport": ">=0.2.0" @@ -20095,6 +20172,12 @@ "requires-port": "^1.0.0" } }, + "node_modules/url-toolkit": { + "version": "2.2.5", + "resolved": "https://registry.npmjs.org/url-toolkit/-/url-toolkit-2.2.5.tgz", + "integrity": "sha512-mtN6xk+Nac+oyJ/PrI7tzfmomRVNFIWKUbG8jdYFt52hxbiReFAXIjYskvu64/dvuW71IcB7lV8l0HvZMac6Jg==", + "license": "Apache-2.0" + }, "node_modules/use": { "version": "3.1.1", "resolved": "https://registry.npmjs.org/use/-/use-3.1.1.tgz", diff --git a/package.json b/package.json index a3dfdd45feff..a8b762450e44 100644 --- a/package.json +++ b/package.json @@ -64,7 +64,7 @@ "exports-loader": "0.6.4", "file-loader": "^6.2.0", "font-awesome": "4.7.0", - "hls.js": "^1.6.15", + "hls.js": "0.14.17", "imports-loader": "0.8.0", "jest-environment-jsdom": "^29.0.0", "jquery": "2.2.4", diff --git a/xmodule/js/spec/video/video_player_spec.js b/xmodule/js/spec/video/video_player_spec.js index 5782d760d830..4845fc80fa1d 100644 --- a/xmodule/js/spec/video/video_player_spec.js +++ b/xmodule/js/spec/video/video_player_spec.js @@ -1,12 +1,12 @@ /* global YT */ // eslint-disable-next-line no-shadow-restricted-names -(function() { +(function(require, define, undefined) { 'use strict'; require( ['video/03_video_player.js', 'hls', 'underscore'], - function(VideoPlayer, Hls, _) { + function(VideoPlayer, HLS, _) { describe('VideoPlayer', function() { var STATUS = window.STATUS, state, @@ -982,7 +982,7 @@ describe('on safari', function() { beforeEach(function() { - spyOn(Hls, 'isSupported').and.returnValue(false); + spyOn(HLS, 'isSupported').and.returnValue(false); state = jasmine.initializeHLSPlayer(); state.canPlayHLS = true; state.browserIsSafari = true; @@ -996,7 +996,7 @@ describe('HLS Video Errors', function() { beforeEach(function() { - spyOn(Hls, 'isSupported').and.returnValue(false); + spyOn(HLS, 'isSupported').and.returnValue(false); state = jasmine.initializeHLSPlayer({sources: ['/base/fixtures/hls/hls.m3u8']}); }); diff --git a/xmodule/js/src/video/02_html5_hls_video.js b/xmodule/js/src/video/02_html5_hls_video.js index 00cc967731f8..cb6a1a2fda27 100644 --- a/xmodule/js/src/video/02_html5_hls_video.js +++ b/xmodule/js/src/video/02_html5_hls_video.js @@ -8,7 +8,7 @@ 'use strict'; define('video/02_html5_hls_video.js', ['underscore', 'video/02_html5_video.js', 'hls'], - function(_, HTML5Video, Hls) { + function(_, HTML5Video, HLS) { var HLSVideo = {}; HLSVideo.Player = (function() { @@ -45,16 +45,16 @@ } else { // load auto start if auto_advance is enabled if (config.state.auto_advance) { - this.hls = new Hls({autoStartLoad: true}); + this.hls = new HLS({autoStartLoad: true}); } else { - this.hls = new Hls({autoStartLoad: false}); + this.hls = new HLS({autoStartLoad: false}); } this.hls.loadSource(config.videoSources[0]); this.hls.attachMedia(this.video); - this.hls.on(Hls.Events.ERROR, this.onError.bind(this)); + this.hls.on(HLS.Events.ERROR, this.onError.bind(this)); - this.hls.on(Hls.Events.MANIFEST_PARSED, function(event, data) { + this.hls.on(HLS.Events.MANIFEST_PARSED, function(event, data) { console.log( '[HLS Video]: MANIFEST_PARSED, qualityLevelsInfo: ', data.levels.map(function(level) { @@ -66,7 +66,7 @@ ); self.config.onReadyHLS(); }); - this.hls.on(Hls.Events.LEVEL_SWITCHED, function(event, data) { + this.hls.on(HLS.Events.LEVEL_SWITCHED, function(event, data) { var level = self.hls.levels[data.level]; console.log( '[HLS Video]: LEVEL_SWITCHED, qualityLevelInfo: ', @@ -114,14 +114,14 @@ Player.prototype.onError = function(event, data) { if (data.fatal) { switch (data.type) { - case Hls.ErrorTypes.NETWORK_ERROR: + case HLS.ErrorTypes.NETWORK_ERROR: console.error( '[HLS Video]: Fatal network error encountered, try to recover. Details: %s', data.details ); this.hls.startLoad(); break; - case Hls.ErrorTypes.MEDIA_ERROR: + case HLS.ErrorTypes.MEDIA_ERROR: console.error( '[HLS Video]: Fatal media error encountered, try to recover. Details: %s', data.details diff --git a/xmodule/js/src/video/03_video_player.js b/xmodule/js/src/video/03_video_player.js index d682a6ccfac7..6215d54689ad 100644 --- a/xmodule/js/src/video/03_video_player.js +++ b/xmodule/js/src/video/03_video_player.js @@ -4,7 +4,7 @@ define( 'video/03_video_player.js', ['video/02_html5_video.js', 'video/02_html5_hls_video.js', 'video/00_resizer.js', 'hls', 'underscore', '../time.js'], - function(HTML5Video, HTML5HLSVideo, Resizer, Hls, _, Time) { + function(HTML5Video, HTML5HLSVideo, Resizer, HLS, _, Time) { var dfd = $.Deferred(), VideoPlayer = function(state) { state.videoPlayer = {}; @@ -157,7 +157,7 @@ // Browser can play HLS videos if either `Media Source Extensions` // feature is supported or browser is safari (native HLS support) - state.canPlayHLS = state.HLSVideoSources.length > 0 && (Hls.isSupported() || state.browserIsSafari); + state.canPlayHLS = state.HLSVideoSources.length > 0 && (HLS.isSupported() || state.browserIsSafari); state.HLSOnlySources = state.config.sources.length > 0 && state.config.sources.length === state.HLSVideoSources.length; From 34bd6597b23daec36845d5604197839440c5547f Mon Sep 17 00:00:00 2001 From: Pradeep Date: Thu, 11 Dec 2025 16:53:17 +0530 Subject: [PATCH 146/351] =?UTF-8?q?Revert=20"Revert=20"fix:=20upgrade=20hl?= =?UTF-8?q?s.js=20to=20version=201.6.15=20and=20update=20related=20impo?= =?UTF-8?q?=E2=80=A6"=20(#62)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This reverts commit f348776bd6440a381216eb54831f8ad001cba95e. --- package-lock.json | 103 ++------------------- package.json | 2 +- xmodule/js/spec/video/video_player_spec.js | 8 +- xmodule/js/src/video/02_html5_hls_video.js | 16 ++-- xmodule/js/src/video/03_video_player.js | 4 +- 5 files changed, 25 insertions(+), 108 deletions(-) diff --git a/package-lock.json b/package-lock.json index 0da387590a61..6db7903427b4 100644 --- a/package-lock.json +++ b/package-lock.json @@ -39,7 +39,7 @@ "exports-loader": "0.6.4", "file-loader": "^6.2.0", "font-awesome": "4.7.0", - "hls.js": "0.14.17", + "hls.js": "^1.6.15", "imports-loader": "0.8.0", "jest-environment-jsdom": "^29.0.0", "jquery": "2.2.4", @@ -3556,15 +3556,12 @@ } }, "node_modules/@keyv/serialize": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/@keyv/serialize/-/serialize-1.0.3.tgz", - "integrity": "sha512-qnEovoOp5Np2JDGonIDL6Ayihw0RhnRh6vxPuHo4RDn1UOzwEo4AeIfpL6UGIrsceWrCMiVPgwRjbHu4vYFc3g==", + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/@keyv/serialize/-/serialize-1.1.1.tgz", + "integrity": "sha512-dXn3FZhPv0US+7dtJsIi2R+c7qWYiReoEh5zUntWCf4oSpMNib8FDhSoed6m3QyZdx5hK7iLFkYk3rNxwt8vTA==", "dev": true, "license": "MIT", - "peer": true, - "dependencies": { - "buffer": "^6.0.3" - } + "peer": true }, "node_modules/@nodelib/fs.scandir": { "version": "2.1.5", @@ -5668,28 +5665,6 @@ "node": ">= 0.6.0" } }, - "node_modules/base64-js": { - "version": "1.5.1", - "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", - "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==", - "dev": true, - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/feross" - }, - { - "type": "patreon", - "url": "https://www.patreon.com/feross" - }, - { - "type": "consulting", - "url": "https://feross.org/support" - } - ], - "license": "MIT", - "peer": true - }, "node_modules/base64id": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/base64id/-/base64id-1.0.0.tgz", @@ -5869,32 +5844,6 @@ "node-int64": "^0.4.0" } }, - "node_modules/buffer": { - "version": "6.0.3", - "resolved": "https://registry.npmjs.org/buffer/-/buffer-6.0.3.tgz", - "integrity": "sha512-FTiCpNxtwiZZHEZbcbTIcZjERVICn9yq/pDFkTl95/AxzD1naBctN7YO68riM/gLSDY7sdrMby8hofADYuuqOA==", - "dev": true, - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/feross" - }, - { - "type": "patreon", - "url": "https://www.patreon.com/feross" - }, - { - "type": "consulting", - "url": "https://feross.org/support" - } - ], - "license": "MIT", - "peer": true, - "dependencies": { - "base64-js": "^1.3.1", - "ieee754": "^1.2.1" - } - }, "node_modules/buffer-alloc": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/buffer-alloc/-/buffer-alloc-1.2.0.tgz", @@ -8703,6 +8652,7 @@ "version": "4.0.7", "resolved": "https://registry.npmjs.org/eventemitter3/-/eventemitter3-4.0.7.tgz", "integrity": "sha512-8guHBZCwKnFhYdHr2ysuRWErTwhoN2X8XELRlrRwpmfeY2jjuUN4taQMsULKUVo1K4DvZl+0pgfyoysHxvmvEw==", + "dev": true, "license": "MIT" }, "node_modules/events": { @@ -10094,14 +10044,10 @@ } }, "node_modules/hls.js": { - "version": "0.14.17", - "resolved": "https://registry.npmjs.org/hls.js/-/hls.js-0.14.17.tgz", - "integrity": "sha512-25A7+m6qqp6UVkuzUQ//VVh2EEOPYlOBg32ypr34bcPO7liBMOkKFvbjbCBfiPAOTA/7BSx1Dujft3Th57WyFg==", - "license": "Apache-2.0", - "dependencies": { - "eventemitter3": "^4.0.3", - "url-toolkit": "^2.1.6" - } + "version": "1.6.15", + "resolved": "https://registry.npmjs.org/hls.js/-/hls.js-1.6.15.tgz", + "integrity": "sha512-E3a5VwgXimGHwpRGV+WxRTKeSp2DW5DI5MWv34ulL3t5UNmyJWCQ1KmLEHbYzcfThfXG8amBL+fCYPneGHC4VA==", + "license": "Apache-2.0" }, "node_modules/hoist-non-react-statics": { "version": "3.3.2", @@ -10341,28 +10287,6 @@ "postcss": "^8.1.0" } }, - "node_modules/ieee754": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.2.1.tgz", - "integrity": "sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==", - "dev": true, - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/feross" - }, - { - "type": "patreon", - "url": "https://www.patreon.com/feross" - }, - { - "type": "consulting", - "url": "https://feross.org/support" - } - ], - "license": "BSD-3-Clause", - "peer": true - }, "node_modules/ignore": { "version": "3.3.10", "resolved": "https://registry.npmjs.org/ignore/-/ignore-3.3.10.tgz", @@ -15439,7 +15363,6 @@ "integrity": "sha512-ijt0LhxWClXBtc1RCt8H0WhlZLAdVX26nWbpsJy+Hblmp81d2F/pFsvlrJhJDDruFHM+ECtxP0H0HzGSrARkwg==", "deprecated": "You or someone you depend on is using Q, the JavaScript Promise library that gave JavaScript developers strong feelings about promises. They can almost certainly migrate to the native JavaScript promise now. Thank you literally everyone for joining me in this bet against the odds. Be excellent to each other.\n\n(For a CapTP with native promises, see @endo/eventual-send and @endo/captp)", "dev": true, - "license": "MIT", "engines": { "node": ">=0.6.0", "teleport": ">=0.2.0" @@ -20172,12 +20095,6 @@ "requires-port": "^1.0.0" } }, - "node_modules/url-toolkit": { - "version": "2.2.5", - "resolved": "https://registry.npmjs.org/url-toolkit/-/url-toolkit-2.2.5.tgz", - "integrity": "sha512-mtN6xk+Nac+oyJ/PrI7tzfmomRVNFIWKUbG8jdYFt52hxbiReFAXIjYskvu64/dvuW71IcB7lV8l0HvZMac6Jg==", - "license": "Apache-2.0" - }, "node_modules/use": { "version": "3.1.1", "resolved": "https://registry.npmjs.org/use/-/use-3.1.1.tgz", diff --git a/package.json b/package.json index a8b762450e44..a3dfdd45feff 100644 --- a/package.json +++ b/package.json @@ -64,7 +64,7 @@ "exports-loader": "0.6.4", "file-loader": "^6.2.0", "font-awesome": "4.7.0", - "hls.js": "0.14.17", + "hls.js": "^1.6.15", "imports-loader": "0.8.0", "jest-environment-jsdom": "^29.0.0", "jquery": "2.2.4", diff --git a/xmodule/js/spec/video/video_player_spec.js b/xmodule/js/spec/video/video_player_spec.js index 4845fc80fa1d..5782d760d830 100644 --- a/xmodule/js/spec/video/video_player_spec.js +++ b/xmodule/js/spec/video/video_player_spec.js @@ -1,12 +1,12 @@ /* global YT */ // eslint-disable-next-line no-shadow-restricted-names -(function(require, define, undefined) { +(function() { 'use strict'; require( ['video/03_video_player.js', 'hls', 'underscore'], - function(VideoPlayer, HLS, _) { + function(VideoPlayer, Hls, _) { describe('VideoPlayer', function() { var STATUS = window.STATUS, state, @@ -982,7 +982,7 @@ describe('on safari', function() { beforeEach(function() { - spyOn(HLS, 'isSupported').and.returnValue(false); + spyOn(Hls, 'isSupported').and.returnValue(false); state = jasmine.initializeHLSPlayer(); state.canPlayHLS = true; state.browserIsSafari = true; @@ -996,7 +996,7 @@ describe('HLS Video Errors', function() { beforeEach(function() { - spyOn(HLS, 'isSupported').and.returnValue(false); + spyOn(Hls, 'isSupported').and.returnValue(false); state = jasmine.initializeHLSPlayer({sources: ['/base/fixtures/hls/hls.m3u8']}); }); diff --git a/xmodule/js/src/video/02_html5_hls_video.js b/xmodule/js/src/video/02_html5_hls_video.js index cb6a1a2fda27..00cc967731f8 100644 --- a/xmodule/js/src/video/02_html5_hls_video.js +++ b/xmodule/js/src/video/02_html5_hls_video.js @@ -8,7 +8,7 @@ 'use strict'; define('video/02_html5_hls_video.js', ['underscore', 'video/02_html5_video.js', 'hls'], - function(_, HTML5Video, HLS) { + function(_, HTML5Video, Hls) { var HLSVideo = {}; HLSVideo.Player = (function() { @@ -45,16 +45,16 @@ } else { // load auto start if auto_advance is enabled if (config.state.auto_advance) { - this.hls = new HLS({autoStartLoad: true}); + this.hls = new Hls({autoStartLoad: true}); } else { - this.hls = new HLS({autoStartLoad: false}); + this.hls = new Hls({autoStartLoad: false}); } this.hls.loadSource(config.videoSources[0]); this.hls.attachMedia(this.video); - this.hls.on(HLS.Events.ERROR, this.onError.bind(this)); + this.hls.on(Hls.Events.ERROR, this.onError.bind(this)); - this.hls.on(HLS.Events.MANIFEST_PARSED, function(event, data) { + this.hls.on(Hls.Events.MANIFEST_PARSED, function(event, data) { console.log( '[HLS Video]: MANIFEST_PARSED, qualityLevelsInfo: ', data.levels.map(function(level) { @@ -66,7 +66,7 @@ ); self.config.onReadyHLS(); }); - this.hls.on(HLS.Events.LEVEL_SWITCHED, function(event, data) { + this.hls.on(Hls.Events.LEVEL_SWITCHED, function(event, data) { var level = self.hls.levels[data.level]; console.log( '[HLS Video]: LEVEL_SWITCHED, qualityLevelInfo: ', @@ -114,14 +114,14 @@ Player.prototype.onError = function(event, data) { if (data.fatal) { switch (data.type) { - case HLS.ErrorTypes.NETWORK_ERROR: + case Hls.ErrorTypes.NETWORK_ERROR: console.error( '[HLS Video]: Fatal network error encountered, try to recover. Details: %s', data.details ); this.hls.startLoad(); break; - case HLS.ErrorTypes.MEDIA_ERROR: + case Hls.ErrorTypes.MEDIA_ERROR: console.error( '[HLS Video]: Fatal media error encountered, try to recover. Details: %s', data.details diff --git a/xmodule/js/src/video/03_video_player.js b/xmodule/js/src/video/03_video_player.js index 6215d54689ad..d682a6ccfac7 100644 --- a/xmodule/js/src/video/03_video_player.js +++ b/xmodule/js/src/video/03_video_player.js @@ -4,7 +4,7 @@ define( 'video/03_video_player.js', ['video/02_html5_video.js', 'video/02_html5_hls_video.js', 'video/00_resizer.js', 'hls', 'underscore', '../time.js'], - function(HTML5Video, HTML5HLSVideo, Resizer, HLS, _, Time) { + function(HTML5Video, HTML5HLSVideo, Resizer, Hls, _, Time) { var dfd = $.Deferred(), VideoPlayer = function(state) { state.videoPlayer = {}; @@ -157,7 +157,7 @@ // Browser can play HLS videos if either `Media Source Extensions` // feature is supported or browser is safari (native HLS support) - state.canPlayHLS = state.HLSVideoSources.length > 0 && (HLS.isSupported() || state.browserIsSafari); + state.canPlayHLS = state.HLSVideoSources.length > 0 && (Hls.isSupported() || state.browserIsSafari); state.HLSOnlySources = state.config.sources.length > 0 && state.config.sources.length === state.HLSVideoSources.length; From f53f4a61aa217874987c5f8910f1b67d3824bf24 Mon Sep 17 00:00:00 2001 From: subhashree-sahu31 Date: Tue, 9 Dec 2025 09:11:37 +0000 Subject: [PATCH 147/351] feat: add waffle flag to enable MFE integration --- lms/envs/common.py | 2 +- openedx/core/djangoapps/user_authn/toggles.py | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/lms/envs/common.py b/lms/envs/common.py index 3dde7156b93e..d563b529949c 100644 --- a/lms/envs/common.py +++ b/lms/envs/common.py @@ -579,7 +579,7 @@ # .. toggle_tickets: 'https://github.com/openedx/edx-platform/pull/24908' # .. toggle_warning: Also set settings.AUTHN_MICROFRONTEND_URL for rollout. This temporary feature # toggle does not have a target removal date. -ENABLE_AUTHN_MICROFRONTEND = os.environ.get("EDXAPP_ENABLE_AUTHN_MFE", False) +ENABLE_AUTHN_MICROFRONTEND = True # .. toggle_name: settings.ENABLE_CATALOG_MICROFRONTEND # .. toggle_implementation: DjangoSetting diff --git a/openedx/core/djangoapps/user_authn/toggles.py b/openedx/core/djangoapps/user_authn/toggles.py index 67da464cbb5f..7bdff08fc563 100644 --- a/openedx/core/djangoapps/user_authn/toggles.py +++ b/openedx/core/djangoapps/user_authn/toggles.py @@ -23,6 +23,10 @@ def should_redirect_to_authn_microfrontend(): return False return configuration_helpers.get_value( 'ENABLE_AUTHN_MICROFRONTEND', settings.FEATURES.get('ENABLE_AUTHN_MICROFRONTEND') + ) and not ( + configuration_helpers.get_value('ENABLE_ENTERPRISE_CUSTOMER', False) and + configuration_helpers.get_value('ENABLE_TPA_HINT_PROVIDER', False) and + configuration_helpers.get_value('ENABLE_SAML_PROVIDER', False) ) From 9c50d4dcc0fd3b68aff71383106e9a76a6e7c240 Mon Sep 17 00:00:00 2001 From: ssurendrannair Date: Wed, 10 Dec 2025 07:19:42 +0000 Subject: [PATCH 148/351] fix: replace hardcoded ENABLE_AUTHN_MICROFRONTEND with env-driven value --- lms/envs/common.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lms/envs/common.py b/lms/envs/common.py index d563b529949c..67f4d92ca83f 100644 --- a/lms/envs/common.py +++ b/lms/envs/common.py @@ -579,7 +579,7 @@ # .. toggle_tickets: 'https://github.com/openedx/edx-platform/pull/24908' # .. toggle_warning: Also set settings.AUTHN_MICROFRONTEND_URL for rollout. This temporary feature # toggle does not have a target removal date. -ENABLE_AUTHN_MICROFRONTEND = True +ENABLE_AUTHN_MICROFRONTEND = os.getenv("EDXAPP_ENABLE_AUTHN_MFE", "false").lower() == "true" # .. toggle_name: settings.ENABLE_CATALOG_MICROFRONTEND # .. toggle_implementation: DjangoSetting From b25d1bcf3137d04e2120e112ccca1eb948dc3657 Mon Sep 17 00:00:00 2001 From: Krish Tyagi Date: Sat, 13 Dec 2025 12:04:47 +0530 Subject: [PATCH 149/351] fix: improving geolite workflow (#63) --- .github/workflows/update-geolite-database.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/update-geolite-database.yml b/.github/workflows/update-geolite-database.yml index c13cd5469722..d9ad767ce57e 100644 --- a/.github/workflows/update-geolite-database.yml +++ b/.github/workflows/update-geolite-database.yml @@ -69,7 +69,7 @@ jobs: - name: Create a branch, commit the code and make a PR id: create-pr run: | - BRANCH="${{ github.actor }}/geoip2-bot-update-country-database-$(echo "${{ github.sha }}" | cut -c 1-7)" + BRANCH="${{ github.actor }}/geoip2-bot-update-country-database-${{ github.run_id }}" git checkout -b $BRANCH git add . git status From 9091801260359a07a465ecc3a1a89a19c2fc032e Mon Sep 17 00:00:00 2001 From: Feanil Patel Date: Thu, 11 Dec 2025 11:18:06 -0500 Subject: [PATCH 150/351] fix: CourseLimitedStaffRole should not be able to access studio. We previously fixed this when the CourseLimitedStaffRole was applied to a course but did not handle the case where the role is applied to a user for a whole org. The underlying issue is that the CourseLimitedStaffRole is a subclass of the CourseStaffRole and much of the system assumes that subclesses are for giving more access not less access. To prevent that from happening for the case of the CourseLimitedStaffRole, when we do CourseStaffRole access checks, we use the strict_role_checking context manager to ensure that we're not accidentally granting the limited_staff role too much access. --- .../contentstore/tests/test_course_listing.py | 44 +++++++++++++++++++ cms/djangoapps/contentstore/views/course.py | 5 ++- common/djangoapps/student/auth.py | 6 ++- common/djangoapps/student/tests/test_authz.py | 18 ++++++++ 4 files changed, 70 insertions(+), 3 deletions(-) diff --git a/cms/djangoapps/contentstore/tests/test_course_listing.py b/cms/djangoapps/contentstore/tests/test_course_listing.py index d256228228cb..990eff83c922 100644 --- a/cms/djangoapps/contentstore/tests/test_course_listing.py +++ b/cms/djangoapps/contentstore/tests/test_course_listing.py @@ -21,8 +21,10 @@ get_courses_accessible_to_user ) from common.djangoapps.course_action_state.models import CourseRerunState +from common.djangoapps.student.models.user import CourseAccessRole from common.djangoapps.student.roles import ( CourseInstructorRole, + CourseLimitedStaffRole, CourseStaffRole, GlobalStaff, OrgInstructorRole, @@ -176,6 +178,48 @@ def test_staff_course_listing(self): with self.assertNumQueries(2): list(_accessible_courses_summary_iter(self.request)) + def test_course_limited_staff_course_listing(self): + # Setup a new course + course_location = self.store.make_course_key('Org', 'CreatedCourse', 'Run') + CourseFactory.create( + org=course_location.org, + number=course_location.course, + run=course_location.run + ) + course = CourseOverviewFactory.create(id=course_location, org=course_location.org) + + # Add the user as a course_limited_staff on the course + CourseLimitedStaffRole(course.id).add_users(self.user) + self.assertTrue(CourseLimitedStaffRole(course.id).has_user(self.user)) + + # Fetch accessible courses list & verify their count + courses_list_by_staff, __ = get_courses_accessible_to_user(self.request) + + # Limited Course Staff should not be able to list courses in Studio + assert len(list(courses_list_by_staff)) == 0 + + def test_org_limited_staff_course_listing(self): + + # Setup a new course + course_location = self.store.make_course_key('Org', 'CreatedCourse', 'Run') + CourseFactory.create( + org=course_location.org, + number=course_location.course, + run=course_location.run + ) + course = CourseOverviewFactory.create(id=course_location, org=course_location.org) + + # Add a user as course_limited_staff on the org + # This is not possible using the course roles classes but is possible via Django admin so we + # insert a row into the model directly to test that scenario. + CourseAccessRole.objects.create(user=self.user, org=course_location.org, role=CourseLimitedStaffRole.ROLE) + + # Fetch accessible courses list & verify their count + courses_list_by_staff, __ = get_courses_accessible_to_user(self.request) + + # Limited Course Staff should not be able to list courses in Studio + assert len(list(courses_list_by_staff)) == 0 + def test_get_course_list_with_invalid_course_location(self): """ Test getting courses with invalid course location (course deleted from modulestore). diff --git a/cms/djangoapps/contentstore/views/course.py b/cms/djangoapps/contentstore/views/course.py index 453e30e0aad0..f965ebb7aaa7 100644 --- a/cms/djangoapps/contentstore/views/course.py +++ b/cms/djangoapps/contentstore/views/course.py @@ -61,6 +61,7 @@ GlobalStaff, UserBasedRole, OrgStaffRole, + strict_role_checking, ) from common.djangoapps.util.json_request import JsonResponse, JsonResponseBadRequest, expect_json from common.djangoapps.util.string_utils import _has_non_ascii_characters @@ -536,7 +537,9 @@ def filter_ccx(course_access): return not isinstance(course_access.course_id, CCXLocator) instructor_courses = UserBasedRole(request.user, CourseInstructorRole.ROLE).courses_with_role() - staff_courses = UserBasedRole(request.user, CourseStaffRole.ROLE).courses_with_role() + with strict_role_checking(): + staff_courses = UserBasedRole(request.user, CourseStaffRole.ROLE).courses_with_role() + all_courses = list(filter(filter_ccx, instructor_courses | staff_courses)) courses_list = [] course_keys = {} diff --git a/common/djangoapps/student/auth.py b/common/djangoapps/student/auth.py index e199142fe377..047f0174a062 100644 --- a/common/djangoapps/student/auth.py +++ b/common/djangoapps/student/auth.py @@ -24,6 +24,7 @@ OrgInstructorRole, OrgLibraryUserRole, OrgStaffRole, + strict_role_checking, ) # Studio permissions: @@ -115,8 +116,9 @@ def get_user_permissions(user, course_key, org=None, service_variant=None): return STUDIO_NO_PERMISSIONS # Staff have all permissions except EDIT_ROLES: - if OrgStaffRole(org=org).has_user(user) or (course_key and user_has_role(user, CourseStaffRole(course_key))): - return STUDIO_VIEW_USERS | STUDIO_EDIT_CONTENT | STUDIO_VIEW_CONTENT + with strict_role_checking(): + if OrgStaffRole(org=org).has_user(user) or (course_key and user_has_role(user, CourseStaffRole(course_key))): + return STUDIO_VIEW_USERS | STUDIO_EDIT_CONTENT | STUDIO_VIEW_CONTENT # Otherwise, for libraries, users can view only: if course_key and isinstance(course_key, LibraryLocator): diff --git a/common/djangoapps/student/tests/test_authz.py b/common/djangoapps/student/tests/test_authz.py index c0b88e6318b5..70636e04b68a 100644 --- a/common/djangoapps/student/tests/test_authz.py +++ b/common/djangoapps/student/tests/test_authz.py @@ -11,6 +11,7 @@ from django.test import TestCase, override_settings from opaque_keys.edx.locator import CourseLocator +from common.djangoapps.student.models.user import CourseAccessRole from common.djangoapps.student.auth import ( add_users, has_studio_read_access, @@ -305,6 +306,23 @@ def test_limited_staff_no_studio_access_cms(self): assert not has_studio_read_access(self.limited_staff, self.course_key) assert not has_studio_write_access(self.limited_staff, self.course_key) + @override_settings(SERVICE_VARIANT='cms') + def test_limited_org_staff_no_studio_access_cms(self): + """ + Verifies that course limited staff have no read and no write access when SERVICE_VARIANT is not 'lms'. + """ + # Add a user as course_limited_staff on the org + # This is not possible using the course roles classes but is possible via Django admin so we + # insert a row into the model directly to test that scenario. + CourseAccessRole.objects.create( + user=self.limited_staff, + org=self.course_key.org, + role=CourseLimitedStaffRole.ROLE, + ) + + assert not has_studio_read_access(self.limited_staff, self.course_key) + assert not has_studio_write_access(self.limited_staff, self.course_key) + class CourseOrgGroupTest(TestCase): """ From dd91e54237faff324bbff4e1f00d5defb26406c6 Mon Sep 17 00:00:00 2001 From: David Ormsbee Date: Fri, 12 Dec 2025 10:02:41 -0500 Subject: [PATCH 151/351] fix: sanitize HTML for course overview & sidebar The "overview" and "about_sidebar_html" fields in the CoursewareInformation view (/api/courseware/course/{courseId}) were returning unsanitized HTML and relying on the client to sanitize it. This commit shifts that work to the server side (clean_dangerous_html) to remove potentially dangerous tags when generating the response. The source of this data is modified in the "Settings and Details" section of a course in Studio. --- .../core/djangoapps/courseware_api/tests/test_views.py | 9 +++++++-- openedx/core/djangoapps/courseware_api/views.py | 9 +++++++-- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/openedx/core/djangoapps/courseware_api/tests/test_views.py b/openedx/core/djangoapps/courseware_api/tests/test_views.py index 1606d245c01f..62fd6d557b34 100644 --- a/openedx/core/djangoapps/courseware_api/tests/test_views.py +++ b/openedx/core/djangoapps/courseware_api/tests/test_views.py @@ -662,15 +662,20 @@ def test_invitation_only_property(self, invitation_only): ) def test_about_sidebar_html_property(self, waffle_enabled, mock_get_course_about_section): """ - Test about_sidebar_html property with different waffle settings + Test about_sidebar_html property with different waffle settings. + + Ensure that when a value is returned,
    ' with override_waffle_switch(ENABLE_COURSE_ABOUT_SIDEBAR_HTML, active=waffle_enabled): meta = self.create_courseware_meta() if waffle_enabled: assert meta.about_sidebar_html == '
    About Course
    ' else: assert meta.about_sidebar_html is None + assert meta.overview == '
    About Course
    ' @ddt.ddt diff --git a/openedx/core/djangoapps/courseware_api/views.py b/openedx/core/djangoapps/courseware_api/views.py index 1dcfc740c84c..a5940d8a132e 100644 --- a/openedx/core/djangoapps/courseware_api/views.py +++ b/openedx/core/djangoapps/courseware_api/views.py @@ -63,6 +63,7 @@ from openedx.core.djangoapps.courseware_api.utils import get_celebrations_dict from openedx.core.djangoapps.enrollments.permissions import ENROLL_IN_COURSE from openedx.core.djangoapps.programs.utils import ProgramProgressMeter +from openedx.core.djangolib.markup import clean_dangerous_html from openedx.core.lib.api.authentication import BearerAuthenticationAllowInactiveUser from openedx.core.lib.api.view_utils import DeveloperErrorViewMixin from openedx.core.lib.courses import get_course_by_id @@ -516,7 +517,9 @@ def about_sidebar_html(self): Returns the HTML content for the course about section. """ if ENABLE_COURSE_ABOUT_SIDEBAR_HTML.is_enabled(): - return get_course_about_section(self.request, self.course, "about_sidebar_html") + return clean_dangerous_html( + get_course_about_section(self.request, self.course, "about_sidebar_html") + ) return None @property @@ -524,7 +527,9 @@ def overview(self): """ Returns the overview HTML content for the course. """ - return get_course_about_section(self.request, self.course, "overview") + return clean_dangerous_html( + get_course_about_section(self.request, self.course, "overview") + ) @method_decorator(transaction.non_atomic_requests, name='dispatch') From f987e8e83469758c28f9bf87780343cb473c28da Mon Sep 17 00:00:00 2001 From: Robert Raposa Date: Wed, 17 Dec 2025 14:30:40 -0500 Subject: [PATCH 152/351] fix: CourseLimitedStaffRole should not be able to access studio. (#65) We previously fixed this when the CourseLimitedStaffRole was applied to a course but did not handle the case where the role is applied to a user for a whole org. The underlying issue is that the CourseLimitedStaffRole is a subclass of the CourseStaffRole and much of the system assumes that subclesses are for giving more access not less access. To prevent that from happening for the case of the CourseLimitedStaffRole, when we do CourseStaffRole access checks, we use the strict_role_checking context manager to ensure that we're not accidentally granting the limited_staff role too much access. Co-authored-by: Feanil Patel (cherry-picked from openedx/edx-platform) --- .../contentstore/tests/test_course_listing.py | 44 +++++++++++++++++++ cms/djangoapps/contentstore/views/course.py | 5 ++- common/djangoapps/student/auth.py | 6 ++- common/djangoapps/student/tests/test_authz.py | 18 ++++++++ 4 files changed, 70 insertions(+), 3 deletions(-) diff --git a/cms/djangoapps/contentstore/tests/test_course_listing.py b/cms/djangoapps/contentstore/tests/test_course_listing.py index e46b493b7b39..a2b6f07d15ef 100644 --- a/cms/djangoapps/contentstore/tests/test_course_listing.py +++ b/cms/djangoapps/contentstore/tests/test_course_listing.py @@ -24,8 +24,10 @@ get_courses_accessible_to_user ) from common.djangoapps.course_action_state.models import CourseRerunState +from common.djangoapps.student.models.user import CourseAccessRole from common.djangoapps.student.roles import ( CourseInstructorRole, + CourseLimitedStaffRole, CourseStaffRole, GlobalStaff, OrgInstructorRole, @@ -188,6 +190,48 @@ def test_staff_course_listing(self): with self.assertNumQueries(2): list(_accessible_courses_summary_iter(self.request)) + def test_course_limited_staff_course_listing(self): + # Setup a new course + course_location = self.store.make_course_key('Org', 'CreatedCourse', 'Run') + CourseFactory.create( + org=course_location.org, + number=course_location.course, + run=course_location.run + ) + course = CourseOverviewFactory.create(id=course_location, org=course_location.org) + + # Add the user as a course_limited_staff on the course + CourseLimitedStaffRole(course.id).add_users(self.user) + self.assertTrue(CourseLimitedStaffRole(course.id).has_user(self.user)) + + # Fetch accessible courses list & verify their count + courses_list_by_staff, __ = get_courses_accessible_to_user(self.request) + + # Limited Course Staff should not be able to list courses in Studio + assert len(list(courses_list_by_staff)) == 0 + + def test_org_limited_staff_course_listing(self): + + # Setup a new course + course_location = self.store.make_course_key('Org', 'CreatedCourse', 'Run') + CourseFactory.create( + org=course_location.org, + number=course_location.course, + run=course_location.run + ) + course = CourseOverviewFactory.create(id=course_location, org=course_location.org) + + # Add a user as course_limited_staff on the org + # This is not possible using the course roles classes but is possible via Django admin so we + # insert a row into the model directly to test that scenario. + CourseAccessRole.objects.create(user=self.user, org=course_location.org, role=CourseLimitedStaffRole.ROLE) + + # Fetch accessible courses list & verify their count + courses_list_by_staff, __ = get_courses_accessible_to_user(self.request) + + # Limited Course Staff should not be able to list courses in Studio + assert len(list(courses_list_by_staff)) == 0 + def test_get_course_list_with_invalid_course_location(self): """ Test getting courses with invalid course location (course deleted from modulestore). diff --git a/cms/djangoapps/contentstore/views/course.py b/cms/djangoapps/contentstore/views/course.py index fa8769dc0cb9..a93a35cf1d3c 100644 --- a/cms/djangoapps/contentstore/views/course.py +++ b/cms/djangoapps/contentstore/views/course.py @@ -56,6 +56,7 @@ GlobalStaff, UserBasedRole, OrgStaffRole, + strict_role_checking, ) from common.djangoapps.util.json_request import JsonResponse, JsonResponseBadRequest, expect_json from common.djangoapps.util.string_utils import _has_non_ascii_characters @@ -536,7 +537,9 @@ def filter_ccx(course_access): return not isinstance(course_access.course_id, CCXLocator) instructor_courses = UserBasedRole(request.user, CourseInstructorRole.ROLE).courses_with_role() - staff_courses = UserBasedRole(request.user, CourseStaffRole.ROLE).courses_with_role() + with strict_role_checking(): + staff_courses = UserBasedRole(request.user, CourseStaffRole.ROLE).courses_with_role() + all_courses = list(filter(filter_ccx, instructor_courses | staff_courses)) courses_list = [] course_keys = {} diff --git a/common/djangoapps/student/auth.py b/common/djangoapps/student/auth.py index e199142fe377..047f0174a062 100644 --- a/common/djangoapps/student/auth.py +++ b/common/djangoapps/student/auth.py @@ -24,6 +24,7 @@ OrgInstructorRole, OrgLibraryUserRole, OrgStaffRole, + strict_role_checking, ) # Studio permissions: @@ -115,8 +116,9 @@ def get_user_permissions(user, course_key, org=None, service_variant=None): return STUDIO_NO_PERMISSIONS # Staff have all permissions except EDIT_ROLES: - if OrgStaffRole(org=org).has_user(user) or (course_key and user_has_role(user, CourseStaffRole(course_key))): - return STUDIO_VIEW_USERS | STUDIO_EDIT_CONTENT | STUDIO_VIEW_CONTENT + with strict_role_checking(): + if OrgStaffRole(org=org).has_user(user) or (course_key and user_has_role(user, CourseStaffRole(course_key))): + return STUDIO_VIEW_USERS | STUDIO_EDIT_CONTENT | STUDIO_VIEW_CONTENT # Otherwise, for libraries, users can view only: if course_key and isinstance(course_key, LibraryLocator): diff --git a/common/djangoapps/student/tests/test_authz.py b/common/djangoapps/student/tests/test_authz.py index c0b88e6318b5..70636e04b68a 100644 --- a/common/djangoapps/student/tests/test_authz.py +++ b/common/djangoapps/student/tests/test_authz.py @@ -11,6 +11,7 @@ from django.test import TestCase, override_settings from opaque_keys.edx.locator import CourseLocator +from common.djangoapps.student.models.user import CourseAccessRole from common.djangoapps.student.auth import ( add_users, has_studio_read_access, @@ -305,6 +306,23 @@ def test_limited_staff_no_studio_access_cms(self): assert not has_studio_read_access(self.limited_staff, self.course_key) assert not has_studio_write_access(self.limited_staff, self.course_key) + @override_settings(SERVICE_VARIANT='cms') + def test_limited_org_staff_no_studio_access_cms(self): + """ + Verifies that course limited staff have no read and no write access when SERVICE_VARIANT is not 'lms'. + """ + # Add a user as course_limited_staff on the org + # This is not possible using the course roles classes but is possible via Django admin so we + # insert a row into the model directly to test that scenario. + CourseAccessRole.objects.create( + user=self.limited_staff, + org=self.course_key.org, + role=CourseLimitedStaffRole.ROLE, + ) + + assert not has_studio_read_access(self.limited_staff, self.course_key) + assert not has_studio_write_access(self.limited_staff, self.course_key) + class CourseOrgGroupTest(TestCase): """ From 4094b3a51b95469b299ac36df9acfd9bf89e29c4 Mon Sep 17 00:00:00 2001 From: Akanshu Aich Date: Fri, 19 Dec 2025 13:23:37 +0530 Subject: [PATCH 153/351] fix: added logs and record exception for datadog monitoring (#68) Ticket: https://2u-internal.atlassian.net/browse/BOMS-290 --- .../djangoapps/user_api/accounts/views.py | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/openedx/core/djangoapps/user_api/accounts/views.py b/openedx/core/djangoapps/user_api/accounts/views.py index 2a5b38fbe3fa..f338fd510177 100644 --- a/openedx/core/djangoapps/user_api/accounts/views.py +++ b/openedx/core/djangoapps/user_api/accounts/views.py @@ -23,6 +23,7 @@ from drf_yasg.utils import swagger_auto_schema from edx_ace import ace from edx_ace.recipient import Recipient +from edx_django_utils.monitoring import record_exception from edx_rest_framework_extensions.auth.jwt.authentication import JwtAuthentication from edx_rest_framework_extensions.auth.session.authentication import SessionAuthenticationAllowInactiveUser from enterprise.models import EnterpriseCourseEnrollment, EnterpriseCustomerUser, PendingEnterpriseCustomerUser @@ -1107,10 +1108,32 @@ def post(self, request): user=retirement.user, ) except UserRetirementStatus.DoesNotExist: + log.error( + 'UserRetirementStatus not found for retirement action' + ) + record_exception() return Response(status=status.HTTP_404_NOT_FOUND) except RetirementStateError as exc: + try: + user_id = retirement.user.id + except AttributeError: + user_id = 'unknown' + log.error( + 'RetirementStateError during user retirement: user_id=%s, error=%s', + user_id, str(exc) + ) + record_exception() return Response(str(exc), status=status.HTTP_400_BAD_REQUEST) except Exception as exc: # pylint: disable=broad-except + try: + user_id = retirement.user.id + except AttributeError: + user_id = 'unknown' + log.error( + 'Unexpected error during user retirement: user_id=%s, error=%s', + user_id, str(exc) + ) + record_exception() return Response(str(exc), status=status.HTTP_500_INTERNAL_SERVER_ERROR) return Response(status=status.HTTP_204_NO_CONTENT) From 670c81f0f2ffb1ef44a9027d7491a5bec59ec52a Mon Sep 17 00:00:00 2001 From: David Ormsbee Date: Wed, 17 Dec 2025 18:32:43 -0500 Subject: [PATCH 154/351] fix: allow library creation by course creators Prior to this, if ENABLE_ORGANIZATION_STAFF_ACCESS_FOR_CONTENT_LIBRARIES was enabled, we would not return the orgs that someone had course creator rights on, even if ENABLE_CREATOR_GROUP was enabled. (For the moment, we are conflating "can create courses" with "can create libraries" for a given org, even though we should probably eventually split those apart.) --- cms/djangoapps/contentstore/views/course.py | 18 +++- .../views/tests/test_organizations.py | 99 +++++++++++++++++++ 2 files changed, 112 insertions(+), 5 deletions(-) diff --git a/cms/djangoapps/contentstore/views/course.py b/cms/djangoapps/contentstore/views/course.py index f965ebb7aaa7..29e890102e73 100644 --- a/cms/djangoapps/contentstore/views/course.py +++ b/cms/djangoapps/contentstore/views/course.py @@ -1843,12 +1843,20 @@ def get_allowed_organizations_for_libraries(user): """ Helper method for returning the list of organizations for which the user is allowed to create libraries. """ + organizations_set = set() + + # This allows org-level staff to create libraries. We should re-evaluate + # whether this is necessary and try to normalize course and library creation + # authorization behavior. if settings.FEATURES.get('ENABLE_ORGANIZATION_STAFF_ACCESS_FOR_CONTENT_LIBRARIES', False): - return get_organizations_for_non_course_creators(user) - elif settings.FEATURES.get('ENABLE_CREATOR_GROUP', False): - return get_organizations(user) - else: - return [] + organizations_set.update(get_organizations_for_non_course_creators(user)) + + # This allows people in the course creator group for an org to create + # libraries, which mimics course behavior. + if settings.FEATURES.get('ENABLE_CREATOR_GROUP', False): + organizations_set.update(get_organizations(user)) + + return sorted(organizations_set) def user_can_create_organizations(user): diff --git a/cms/djangoapps/contentstore/views/tests/test_organizations.py b/cms/djangoapps/contentstore/views/tests/test_organizations.py index cf3a376f3461..d231e3adc507 100644 --- a/cms/djangoapps/contentstore/views/tests/test_organizations.py +++ b/cms/djangoapps/contentstore/views/tests/test_organizations.py @@ -3,12 +3,18 @@ import json +from django.conf import settings from django.test import TestCase +from django.test.utils import override_settings from django.urls import reverse from organizations.api import add_organization +from cms.djangoapps.course_creators.models import CourseCreator +from common.djangoapps.student.roles import OrgStaffRole from common.djangoapps.student.tests.factories import UserFactory +from ..course import get_allowed_organizations_for_libraries + class TestOrganizationListing(TestCase): """Verify Organization listing behavior.""" @@ -32,3 +38,96 @@ def test_organization_list(self): self.assertEqual(response.status_code, 200) org_names = json.loads(response.content.decode('utf-8')) self.assertEqual(org_names, self.org_short_names) + + +class TestOrganizationsForLibraries(TestCase): + """ + Verify who is allowed to create Libraries. + + This uses some low-level implementation details to set up course creator and + org staff data, which should be replaced by API calls. + + The behavior of this call depends on two FEATURES toggles: + + * ENABLE_ORGANIZATION_STAFF_ACCESS_FOR_CONTENT_LIBRARIES + * ENABLE_CREATOR_GROUP + """ + + @classmethod + def setUpTestData(cls): + cls.library_author = UserFactory(is_staff=False) + cls.org_short_names = ["OrgStaffOrg", "CreatorOrg", "RandomOrg"] + cls.orgs = {} + for index, short_name in enumerate(cls.org_short_names): + cls.orgs[short_name] = add_organization(organization_data={ + 'name': 'Test Organization %s' % index, + 'short_name': short_name, + 'description': 'Testing Organization %s Description' % index, + }) + + # Our user is an org staff for OrgStaffOrg + OrgStaffRole("OrgStaffOrg").add_users(cls.library_author) + + # Our user is also a CourseCreator in CreatorOrg + creator = CourseCreator.objects.create( + user=cls.library_author, + state=CourseCreator.GRANTED, + all_organizations=False, + ) + # The following is because course_creators app logic assumes that all + # updates to CourseCreator go through the CourseCreatorAdmin. + # Specifically, CourseCreatorAdmin.save_model() attaches the current + # request.user to the model instance's .admin field, and then the + # course_creator_organizations_changed_callback() signal handler assumes + # creator.admin is present. I think that code could use some judicious + # refactoring, but I'm just writing this test as part of a last-minute + # Ulmo bug fix, and I don't want to add risk by refactoring something as + # critical-path as course_creators as part of this work. + creator.admin = UserFactory(is_staff=True) + creator.organizations.add( + cls.orgs["CreatorOrg"]['id'] + ) + + @override_settings( + FEATURES={ + **settings.FEATURES, + 'ENABLE_ORGANIZATION_STAFF_ACCESS_FOR_CONTENT_LIBRARIES': False, + 'ENABLE_CREATOR_GROUP': False, + } + ) + def test_both_toggles_disabled(self): + allowed_orgs = get_allowed_organizations_for_libraries(self.library_author) + assert allowed_orgs == [] + + @override_settings( + FEATURES={ + **settings.FEATURES, + 'ENABLE_ORGANIZATION_STAFF_ACCESS_FOR_CONTENT_LIBRARIES': True, + 'ENABLE_CREATOR_GROUP': True, + } + ) + def test_both_toggles_enabled(self): + allowed_orgs = get_allowed_organizations_for_libraries(self.library_author) + assert allowed_orgs == ["CreatorOrg", "OrgStaffOrg"] + + @override_settings( + FEATURES={ + **settings.FEATURES, + 'ENABLE_ORGANIZATION_STAFF_ACCESS_FOR_CONTENT_LIBRARIES': True, + 'ENABLE_CREATOR_GROUP': False, + } + ) + def test_org_staff_enabled(self): + allowed_orgs = get_allowed_organizations_for_libraries(self.library_author) + assert allowed_orgs == ["OrgStaffOrg"] + + @override_settings( + FEATURES={ + **settings.FEATURES, + 'ENABLE_ORGANIZATION_STAFF_ACCESS_FOR_CONTENT_LIBRARIES': False, + 'ENABLE_CREATOR_GROUP': True, + } + ) + def test_creator_group_enabled(self): + allowed_orgs = get_allowed_organizations_for_libraries(self.library_author) + assert allowed_orgs == ["CreatorOrg"] From 30f0dab4e9942db6fed453adf17cc31dd50567f7 Mon Sep 17 00:00:00 2001 From: Vivek Date: Fri, 19 Dec 2025 22:12:36 +0530 Subject: [PATCH 155/351] feat: extend beta component types and improve editor behavior (#67) * feat: add games to beta component types * feat: auto-open editor on block creation --------- Co-authored-by: pganesh-apphelix --- cms/djangoapps/contentstore/views/component.py | 6 ++++-- cms/static/js/views/pages/container.js | 6 ++++++ 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/cms/djangoapps/contentstore/views/component.py b/cms/djangoapps/contentstore/views/component.py index 470d4274d1b0..82db51f958fe 100644 --- a/cms/djangoapps/contentstore/views/component.py +++ b/cms/djangoapps/contentstore/views/component.py @@ -69,11 +69,13 @@ def is_games_xblock_enabled(): 'openassessment', 'drag-and-drop-v2', ] -if is_games_xblock_enabled(): - COMPONENT_TYPES.append('games') BETA_COMPONENT_TYPES = ['library_v2', 'itembank'] +if is_games_xblock_enabled(): + COMPONENT_TYPES.append('games') + BETA_COMPONENT_TYPES.append('games') + ADVANCED_COMPONENT_TYPES = sorted({name for name, class_ in XBlock.load_classes()} - set(COMPONENT_TYPES)) ADVANCED_PROBLEM_TYPES = settings.ADVANCED_PROBLEM_TYPES diff --git a/cms/static/js/views/pages/container.js b/cms/static/js/views/pages/container.js index 9f8c5ddc6d51..88a0f3397f2a 100644 --- a/cms/static/js/views/pages/container.js +++ b/cms/static/js/views/pages/container.js @@ -301,6 +301,12 @@ function($, _, Backbone, gettext, BasePage, this.xblockView.refresh(xblockView, block_added, is_duplicate); // Update publish and last modified information from the server. this.model.fetch(); + + // Auto-open editor for games blocks when first added + if (block_added && !is_duplicate && xblockView && xblockView.$el && + xblockView.$el.find('.xblock-header-primary').attr('data-block-type') === 'games') { + setTimeout(function() { xblockView.$el.find('.edit-button').first().trigger('click'); }, 500); + } }, renderAddXBlockComponents: function() { From e3084cf4c3752d07edd39ea1642f2f121dc7b04d Mon Sep 17 00:00:00 2001 From: Feanil Patel Date: Fri, 19 Dec 2025 10:51:58 -0500 Subject: [PATCH 156/351] build: Don't update common_constraints.txt on re-compilation. Re-compilation and upgrade-package should be able to run without updating the common_constraints.txt file. We do this all the time when backporting fixes to older releases. We shouldn't pull in the latest common_constraints.txt in those cases as they may not be compatible with older releases. --- Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 92a2e37b9aac..ed6d1f2b7a41 100644 --- a/Makefile +++ b/Makefile @@ -116,7 +116,7 @@ $(COMMON_CONSTRAINTS_TXT): printf "$(COMMON_CONSTRAINTS_TEMP_COMMENT)" | cat - $(@) > temp && mv temp $(@) compile-requirements: export CUSTOM_COMPILE_COMMAND=make upgrade -compile-requirements: pre-requirements $(COMMON_CONSTRAINTS_TXT) ## Re-compile *.in requirements to *.txt +compile-requirements: pre-requirements ## Re-compile *.in requirements to *.txt @# Bootstrapping: Rebuild pip and pip-tools first, and then install them @# so that if there are any failures we'll know now, rather than the next @# time someone tries to use the outputs. @@ -139,7 +139,7 @@ compile-requirements: pre-requirements $(COMMON_CONSTRAINTS_TXT) ## Re-compile * export REBUILD=''; \ done -upgrade: ## update the pip requirements files to use the latest releases satisfying our constraints +upgrade: $(COMMON_CONSTRAINTS_TXT) ## update the pip requirements files to use the latest releases satisfying our constraints $(MAKE) compile-requirements COMPILE_OPTS="--upgrade" upgrade-package: ## update just one package to the latest usable release From 0237bfa2bccb0241787bae2e43d51855a8ab3179 Mon Sep 17 00:00:00 2001 From: Daniel Wong Date: Fri, 5 Dec 2025 14:13:30 -0600 Subject: [PATCH 157/351] fix: bump learning-core to 0.30.2 --- requirements/constraints.txt | 2 +- requirements/edx/base.txt | 2 +- requirements/edx/development.txt | 2 +- requirements/edx/doc.txt | 2 +- requirements/edx/testing.txt | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/requirements/constraints.txt b/requirements/constraints.txt index 6b5f3e38c2ed..dd80bc9c72c1 100644 --- a/requirements/constraints.txt +++ b/requirements/constraints.txt @@ -61,7 +61,7 @@ numpy<2.0.0 # Date: 2023-09-18 # pinning this version to avoid updates while the library is being developed # Issue for unpinning: https://github.com/openedx/edx-platform/issues/35269 -openedx-learning==0.30.1 +openedx-learning==0.30.2 # Date: 2023-11-29 # Open AI version 1.0.0 dropped support for openai.ChatCompletion which is currently in use in enterprise. diff --git a/requirements/edx/base.txt b/requirements/edx/base.txt index 60e19cd22853..27047d2f9c17 100644 --- a/requirements/edx/base.txt +++ b/requirements/edx/base.txt @@ -856,7 +856,7 @@ openedx-filters==2.1.0 # ora2 openedx-forum==0.3.8 # via -r requirements/edx/kernel.in -openedx-learning==0.30.1 +openedx-learning==0.30.2 # via # -c requirements/constraints.txt # -r requirements/edx/kernel.in diff --git a/requirements/edx/development.txt b/requirements/edx/development.txt index 85ac66aed195..e245b5c98574 100644 --- a/requirements/edx/development.txt +++ b/requirements/edx/development.txt @@ -1420,7 +1420,7 @@ openedx-forum==0.3.8 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt -openedx-learning==0.30.1 +openedx-learning==0.30.2 # via # -c requirements/constraints.txt # -r requirements/edx/doc.txt diff --git a/requirements/edx/doc.txt b/requirements/edx/doc.txt index 68c115676fe9..2c79877fc7ab 100644 --- a/requirements/edx/doc.txt +++ b/requirements/edx/doc.txt @@ -1034,7 +1034,7 @@ openedx-filters==2.1.0 # ora2 openedx-forum==0.3.8 # via -r requirements/edx/base.txt -openedx-learning==0.30.1 +openedx-learning==0.30.2 # via # -c requirements/constraints.txt # -r requirements/edx/base.txt diff --git a/requirements/edx/testing.txt b/requirements/edx/testing.txt index f58ef16f9e31..63f989732fe6 100644 --- a/requirements/edx/testing.txt +++ b/requirements/edx/testing.txt @@ -1080,7 +1080,7 @@ openedx-filters==2.1.0 # ora2 openedx-forum==0.3.8 # via -r requirements/edx/base.txt -openedx-learning==0.30.1 +openedx-learning==0.30.2 # via # -c requirements/constraints.txt # -r requirements/edx/base.txt From a0c83f7af745407e1ce6a3c33f8c6fb76ab89f89 Mon Sep 17 00:00:00 2001 From: Kyle McCormick Date: Fri, 19 Dec 2025 14:30:34 -0500 Subject: [PATCH 158/351] fix: Various fixes to modulestore_migrator [ulmo backport] (#37796) For legacy library_content references in courses, this PR: - **Removes the spurious sync after updating a reference to a migrated library**, so that users don't need to "update" their content _after_ updating their reference, _unless_ there were real content edits that happened since they last synced. We do this by correctly associating a DraftChangeLogRecord with the ModulestoreBlockSource migration artifact, and then comparing that version information before offering a sync. (related issue: https://github.com/openedx/frontend-app-authoring/issues/2626). - **Prompts users to update a reference to a migrated library with higher priority than prompting them to sync legacy content updates for that reference**, so that users don't end up needing to accept legacy content updates in order to get a to a point where they can update to V2 content. - **Ensures the library references in courses always follow the correct migration,** as defined by the data `forwarded` fields in the data model, which are populated based on the REST API spec and the stated product UI requirements. For the migration itself, this PR: - **Allows non-admins to migrate libraries**, fixing: https://github.com/openedx/edx-platform/issues/37774 - **When triggered via the UI, ensures the migration uses nice title-based target slugs instead of ugly source-hash-based slugs.** We've had this as an option for a long time, but preserve_url_slugs defaulted to True instead of False in the REST API serializer, so we weren't taking advantage of it. - **Unifies logic between single-source and bulk migration**. These were implement as two separate code paths, with drift in their implementations. In particular, the collection update-vs-create-new logic was completely different for single-souce vs. bulk. - **When using the Skip or Update strategies for repeats, it consistently follows mappings established by the latest successful migration** rather than following mappings across arbitrary previous migrations. - **We log unexpected exceptions more often**, although there is so much more room for improvement here. - **Adds more validation to the REST API** so that client mistakes more often become 400s with validation messages rather than 500s. For developers, this PR: - Adds unit tests to the REST API - Ensures that all migration business logic now goes through a general-purpose Python API. - Ensures that the data model (specifically `forwarded`, and `change_log_record`) is now populated and respected. - Adds more type annotations. Backports: 91e521ef510160a2bcbc87e24002cca70cd3c34f Backport note: Compared to the original commit, this backport commit excludes the REST APIs which were not defined at the time of the Ulmo cutoff: * /api/v1/modulestore_migrator/libraries * /api/v1/modulestore_migrator/migration_info * /api/v1/modulestore_migrator/migration_blocks --- .../rest_api/v1/serializers/home.py | 25 +- .../contentstore/rest_api/v1/views/home.py | 2 +- .../rest_api/v1/views/tests/test_home.py | 47 +- cms/djangoapps/contentstore/utils.py | 31 +- cms/djangoapps/contentstore/views/course.py | 17 +- cms/djangoapps/modulestore_migrator/admin.py | 5 +- cms/djangoapps/modulestore_migrator/api.py | 163 --- .../modulestore_migrator/api/__init__.py | 9 + .../modulestore_migrator/api/read_api.py | 244 ++++ .../modulestore_migrator/api/write_api.py | 94 ++ cms/djangoapps/modulestore_migrator/data.py | 62 + ...estoreblockmigration_unsupported_reason.py | 32 + ...dulestoreblocksource_forwarded_and_more.py | 30 + cms/djangoapps/modulestore_migrator/models.py | 45 +- .../rest_api/v1/serializers.py | 122 +- .../modulestore_migrator/rest_api/v1/urls.py | 7 +- .../modulestore_migrator/rest_api/v1/views.py | 131 +- cms/djangoapps/modulestore_migrator/tasks.py | 684 +++------ .../modulestore_migrator/tests/test_api.py | 348 +++-- .../tests/test_rest_api.py | 822 +++++++++++ .../modulestore_migrator/tests/test_tasks.py | 1223 ++++++----------- .../content_libraries/api/blocks.py | 2 +- setup.cfg | 1 + xmodule/library_content_block.py | 37 +- xmodule/tests/test_library_content.py | 49 +- 25 files changed, 2527 insertions(+), 1705 deletions(-) delete mode 100644 cms/djangoapps/modulestore_migrator/api.py create mode 100644 cms/djangoapps/modulestore_migrator/api/__init__.py create mode 100644 cms/djangoapps/modulestore_migrator/api/read_api.py create mode 100644 cms/djangoapps/modulestore_migrator/api/write_api.py create mode 100644 cms/djangoapps/modulestore_migrator/migrations/0004_alter_modulestoreblockmigration_target_squashed_0005_modulestoreblockmigration_unsupported_reason.py create mode 100644 cms/djangoapps/modulestore_migrator/migrations/0006_alter_modulestoreblocksource_forwarded_and_more.py create mode 100644 cms/djangoapps/modulestore_migrator/tests/test_rest_api.py diff --git a/cms/djangoapps/contentstore/rest_api/v1/serializers/home.py b/cms/djangoapps/contentstore/rest_api/v1/serializers/home.py index bbc45ddf9a37..8ee8cb035478 100644 --- a/cms/djangoapps/contentstore/rest_api/v1/serializers/home.py +++ b/cms/djangoapps/contentstore/rest_api/v1/serializers/home.py @@ -28,26 +28,11 @@ class LibraryViewSerializer(serializers.Serializer): org = serializers.CharField() number = serializers.CharField() can_edit = serializers.BooleanField() - is_migrated = serializers.SerializerMethodField() - migrated_to_title = serializers.CharField( - source="migrations__target__title", - required=False - ) - migrated_to_key = serializers.CharField( - source="migrations__target__key", - required=False - ) - migrated_to_collection_key = serializers.CharField( - source="migrations__target_collection__key", - required=False - ) - migrated_to_collection_title = serializers.CharField( - source="migrations__target_collection__title", - required=False - ) - - def get_is_migrated(self, obj): - return "migrations__target__key" in obj + is_migrated = serializers.BooleanField() + migrated_to_title = serializers.CharField(required=False) + migrated_to_key = serializers.CharField(required=False) + migrated_to_collection_key = serializers.CharField(required=False) + migrated_to_collection_title = serializers.CharField(required=False) class CourseHomeTabSerializer(serializers.Serializer): diff --git a/cms/djangoapps/contentstore/rest_api/v1/views/home.py b/cms/djangoapps/contentstore/rest_api/v1/views/home.py index a4e93de9caff..95723020c11f 100644 --- a/cms/djangoapps/contentstore/rest_api/v1/views/home.py +++ b/cms/djangoapps/contentstore/rest_api/v1/views/home.py @@ -236,7 +236,7 @@ def get(self, request: Request): "number": "CPSPR", "can_edit": true } - ], } + ], ``` """ diff --git a/cms/djangoapps/contentstore/rest_api/v1/views/tests/test_home.py b/cms/djangoapps/contentstore/rest_api/v1/views/tests/test_home.py index cd7592c46629..72d58fa00dfa 100644 --- a/cms/djangoapps/contentstore/rest_api/v1/views/tests/test_home.py +++ b/cms/djangoapps/contentstore/rest_api/v1/views/tests/test_home.py @@ -18,7 +18,6 @@ from cms.djangoapps.contentstore.tests.utils import CourseTestCase from cms.djangoapps.modulestore_migrator import api as migrator_api from cms.djangoapps.modulestore_migrator.data import CompositionLevel, RepeatHandlingStrategy -from cms.djangoapps.modulestore_migrator.tests.factories import ModulestoreSourceFactory from openedx.core.djangoapps.content.course_overviews.tests.factories import CourseOverviewFactory from openedx.core.djangoapps.content_libraries import api as lib_api @@ -253,8 +252,9 @@ class HomePageLibrariesViewTest(LibraryTestCase): def setUp(self): super().setUp() - # Create an additional legacy library + # Create an two additional legacy libaries self.lib_key_1 = self._create_library(library="lib1") + self.lib_key_2 = self._create_library(library="lib2") self.organization = OrganizationFactory() # Create a new v2 library @@ -269,7 +269,6 @@ def setUp(self): library = lib_api.ContentLibrary.objects.get(slug=self.lib_key_v2.slug) learning_package = library.learning_package # Create a migration source for the legacy library - self.source = ModulestoreSourceFactory(key=self.lib_key_1) self.url = reverse("cms.djangoapps.contentstore:v1:libraries") # Create a collection to migrate this library to collection_key = "test-collection" @@ -280,20 +279,32 @@ def setUp(self): created_by=self.user.id, ) - # Migrate self.lib_key_1 to self.lib_key_v2 + # Migrate both lib_key_1 and lib_key_2 to v2 + # Only make lib_key_1 a "forwarding" migration. migrator_api.start_migration_to_library( user=self.user, - source_key=self.source.key, + source_key=self.lib_key_1, target_library_key=self.lib_key_v2, target_collection_slug=collection_key, - composition_level=CompositionLevel.Component.value, - repeat_handling_strategy=RepeatHandlingStrategy.Skip.value, + composition_level=CompositionLevel.Component, + repeat_handling_strategy=RepeatHandlingStrategy.Skip, + preserve_url_slugs=True, + forward_source_to_target=True, + ) + migrator_api.start_migration_to_library( + user=self.user, + source_key=self.lib_key_2, + target_library_key=self.lib_key_v2, + target_collection_slug=collection_key, + composition_level=CompositionLevel.Component, + repeat_handling_strategy=RepeatHandlingStrategy.Skip, preserve_url_slugs=True, forward_source_to_target=False, ) def test_home_page_libraries_response(self): - """Check successful response content""" + """Check sucessful response content""" + self.maxDiff = None response = self.client.get(self.url) expected_response = { @@ -322,6 +333,17 @@ def test_home_page_libraries_response(self): 'migrated_to_collection_key': 'test-collection', 'migrated_to_collection_title': 'Test Collection', }, + # Third library was migrated, but not with forwarding. + # So, it appears just like the unmigrated library. + { + 'display_name': 'Test Library', + 'library_key': 'library-v1:org+lib2', + 'url': '/library/library-v1:org+lib2', + 'org': 'org', + 'number': 'lib2', + 'can_edit': True, + 'is_migrated': False, + }, ] } @@ -366,6 +388,15 @@ def test_home_page_libraries_response(self): 'can_edit': True, 'is_migrated': False, }, + { + 'display_name': 'Test Library', + 'library_key': 'library-v1:org+lib2', + 'url': '/library/library-v1:org+lib2', + 'org': 'org', + 'number': 'lib2', + 'can_edit': True, + 'is_migrated': False, + }, ], } diff --git a/cms/djangoapps/contentstore/utils.py b/cms/djangoapps/contentstore/utils.py index 78e41b7b1813..5506d8c33e41 100644 --- a/cms/djangoapps/contentstore/utils.py +++ b/cms/djangoapps/contentstore/utils.py @@ -57,7 +57,8 @@ ) from cms.djangoapps.models.settings.course_grading import CourseGradingModel from cms.djangoapps.models.settings.course_metadata import CourseMetadata -from cms.djangoapps.modulestore_migrator.api import get_migration_info +from cms.djangoapps.modulestore_migrator import api as migrator_api +from cms.djangoapps.modulestore_migrator.data import ModulestoreMigration from common.djangoapps.course_action_state.managers import CourseActionStateItemNotFoundError from common.djangoapps.course_action_state.models import CourseRerunState, CourseRerunUIStateManager from common.djangoapps.course_modes.models import CourseMode @@ -1578,13 +1579,12 @@ def request_response_format_is_json(request, response_format): def get_library_context(request, request_is_json=False): """ - Utils is used to get context of course home library tab. - It is used for both DRF and django views. + Utils is used to get context of course home library tab. Returned in DRF view. """ from cms.djangoapps.contentstore.views.course import ( _accessible_libraries_iter, - _format_library_for_view, _get_course_creator_status, + format_library_for_view, get_allowed_organizations, get_allowed_organizations_for_libraries, user_can_create_organizations, @@ -1596,21 +1596,25 @@ def get_library_context(request, request_is_json=False): user_can_create_library, ) + is_migrated: bool | None # None means: do not filter on is_migrated + if (is_migrated_param := request.GET.get('is_migrated')) is not None: + is_migrated = BooleanField().to_internal_value(is_migrated_param) + else: + is_migrated = None libraries = list(_accessible_libraries_iter(request.user) if libraries_v1_enabled() else []) - library_keys = [lib.location.library_key for lib in libraries] - migration_info = get_migration_info(library_keys) - is_migrated_filter = request.GET.get('is_migrated', None) + migration_info: dict[LibraryLocator, ModulestoreMigration | None] = { + lib.id: migrator_api.get_forwarding(lib.id) + for lib in libraries + } data = { 'libraries': [ - _format_library_for_view( + format_library_for_view( lib, request, - migrated_to=migration_info.get(lib.location.library_key) + migration=migration_info[lib.id], ) for lib in libraries - if is_migrated_filter is None or ( - BooleanField().to_internal_value(is_migrated_filter) == (lib.location.library_key in migration_info) - ) + if is_migrated is None or is_migrated == bool(migration_info[lib.id]) ] } @@ -1719,8 +1723,7 @@ def format_in_process_course_view(uca): def get_home_context(request, no_course=False): """ - Utils is used to get context of course home. - It is used for both DRF and django views. + Utils is used to get context of course home. Returned by DRF view. """ from cms.djangoapps.contentstore.views.course import ( diff --git a/cms/djangoapps/contentstore/views/course.py b/cms/djangoapps/contentstore/views/course.py index 29e890102e73..681ea5f9fe9b 100644 --- a/cms/djangoapps/contentstore/views/course.py +++ b/cms/djangoapps/contentstore/views/course.py @@ -7,7 +7,7 @@ import random import re import string -from typing import Dict, NamedTuple, Optional +from typing import Dict import django.utils from ccx_keys.locator import CCXLocator @@ -44,6 +44,7 @@ from cms.djangoapps.models.settings.course_grading import CourseGradingModel from cms.djangoapps.models.settings.course_metadata import CourseMetadata from cms.djangoapps.models.settings.encoder import CourseSettingsEncoder +from cms.djangoapps.modulestore_migrator.data import ModulestoreMigration from cms.djangoapps.contentstore.api.views.utils import get_bool_param from common.djangoapps.course_action_state.managers import CourseActionStateItemNotFoundError from common.djangoapps.course_action_state.models import CourseRerunState, CourseRerunUIStateManager @@ -674,11 +675,18 @@ def library_listing(request): ) -def _format_library_for_view(library, request, migrated_to: Optional[NamedTuple]): +def format_library_for_view(library, request, migration: ModulestoreMigration | None): """ Return a dict of the data which the view requires for each library """ - + migration_info = {} + if migration: + migration_info = { + 'migrated_to_key': migration.target_key, + 'migrated_to_title': migration.target_title, + 'migrated_to_collection_key': migration.target_collection_slug, + 'migrated_to_collection_title': migration.target_collection_title, + } return { 'display_name': library.display_name, 'library_key': str(library.location.library_key), @@ -686,7 +694,8 @@ def _format_library_for_view(library, request, migrated_to: Optional[NamedTuple] 'org': library.display_org_with_default, 'number': library.display_number_with_default, 'can_edit': has_studio_write_access(request.user, library.location.library_key), - **(migrated_to._asdict() if migrated_to is not None else {}), + 'is_migrated': migration is not None, + **migration_info, } diff --git a/cms/djangoapps/modulestore_migrator/admin.py b/cms/djangoapps/modulestore_migrator/admin.py index 8eef778531ac..c9a5c90256fd 100644 --- a/cms/djangoapps/modulestore_migrator/admin.py +++ b/cms/djangoapps/modulestore_migrator/admin.py @@ -147,8 +147,8 @@ def start_migration_task( source_key=source.key, target_library_key=target_library_key, target_collection_slug=target_collection_slug, - composition_level=form.cleaned_data['composition_level'], - repeat_handling_strategy=form.cleaned_data['repeat_handling_strategy'], + composition_level=CompositionLevel(form.cleaned_data['composition_level']), + repeat_handling_strategy=RepeatHandlingStrategy(form.cleaned_data['repeat_handling_strategy']), preserve_url_slugs=form.cleaned_data['preserve_url_slugs'], forward_source_to_target=form.cleaned_data['forward_to_target'], ) @@ -178,6 +178,7 @@ class ModulestoreBlockMigrationInline(admin.TabularInline): "source", "target", "change_log_record", + "unsupported_reason", ) list_display = ("id", *readonly_fields) diff --git a/cms/djangoapps/modulestore_migrator/api.py b/cms/djangoapps/modulestore_migrator/api.py deleted file mode 100644 index e16d3061c9d8..000000000000 --- a/cms/djangoapps/modulestore_migrator/api.py +++ /dev/null @@ -1,163 +0,0 @@ -""" -API for migration from modulestore to learning core -""" -from celery.result import AsyncResult -from opaque_keys import InvalidKeyError -from opaque_keys.edx.keys import CourseKey, LearningContextKey, UsageKey -from opaque_keys.edx.locator import LibraryLocator, LibraryLocatorV2, LibraryUsageLocatorV2 -from openedx_learning.api.authoring import get_collection -from openedx_learning.api.authoring_models import Component -from user_tasks.models import UserTaskStatus - -from openedx.core.djangoapps.content_libraries.api import get_library, library_component_usage_key -from openedx.core.types.user import AuthUser - -from . import tasks -from .models import ModulestoreBlockMigration, ModulestoreSource - -__all__ = ( - "start_migration_to_library", - "start_bulk_migration_to_library", - "is_successfully_migrated", - "get_migration_info", - "get_target_block_usage_keys", -) - - -def start_migration_to_library( - *, - user: AuthUser, - source_key: LearningContextKey, - target_library_key: LibraryLocatorV2, - target_collection_slug: str | None = None, - composition_level: str, - repeat_handling_strategy: str, - preserve_url_slugs: bool, - forward_source_to_target: bool, -) -> AsyncResult: - """ - Import a course or legacy library into a V2 library (or, a collection within a V2 library). - """ - source, _ = ModulestoreSource.objects.get_or_create(key=source_key) - target_library = get_library(target_library_key) - # get_library ensures that the library is connected to a learning package. - target_package_id: int = target_library.learning_package_id # type: ignore[assignment] - target_collection_id = None - - if target_collection_slug: - target_collection_id = get_collection(target_package_id, target_collection_slug).id - - return tasks.migrate_from_modulestore.delay( - user_id=user.id, - source_pk=source.id, - target_library_key=str(target_library_key), - target_collection_pk=target_collection_id, - composition_level=composition_level, - repeat_handling_strategy=repeat_handling_strategy, - preserve_url_slugs=preserve_url_slugs, - forward_source_to_target=forward_source_to_target, - ) - - -def start_bulk_migration_to_library( - *, - user: AuthUser, - source_key_list: list[LearningContextKey], - target_library_key: LibraryLocatorV2, - target_collection_slug_list: list[str | None] | None = None, - create_collections: bool = False, - composition_level: str, - repeat_handling_strategy: str, - preserve_url_slugs: bool, - forward_source_to_target: bool, -) -> AsyncResult: - """ - Import a list of courses or legacy libraries into a V2 library (or, a collections within a V2 library). - """ - target_library = get_library(target_library_key) - # get_library ensures that the library is connected to a learning package. - target_package_id: int = target_library.learning_package_id # type: ignore[assignment] - - sources_pks: list[int] = [] - for source_key in source_key_list: - source, _ = ModulestoreSource.objects.get_or_create(key=source_key) - sources_pks.append(source.id) - - target_collection_pks: list[int | None] = [] - if target_collection_slug_list: - for target_collection_slug in target_collection_slug_list: - if target_collection_slug: - target_collection_id = get_collection(target_package_id, target_collection_slug).id - target_collection_pks.append(target_collection_id) - else: - target_collection_pks.append(None) - - return tasks.bulk_migrate_from_modulestore.delay( - user_id=user.id, - sources_pks=sources_pks, - target_library_key=str(target_library_key), - target_collection_pks=target_collection_pks, - create_collections=create_collections, - composition_level=composition_level, - repeat_handling_strategy=repeat_handling_strategy, - preserve_url_slugs=preserve_url_slugs, - forward_source_to_target=forward_source_to_target, - ) - - -def is_successfully_migrated( - source_key: CourseKey | LibraryLocator, - source_version: str | None = None, -) -> bool: - """ - Check if the source course/library has been migrated successfully. - """ - filters = {"task_status__state": UserTaskStatus.SUCCEEDED} - if source_version is not None: - filters["source_version"] = source_version - return ModulestoreSource.objects.get_or_create(key=str(source_key))[0].migrations.filter(**filters).exists() - - -def get_migration_info(source_keys: list[CourseKey | LibraryLocator]) -> dict: - """ - Check if the source course/library has been migrated successfully and return target info - """ - return { - info.key: info - for info in ModulestoreSource.objects.filter( - migrations__task_status__state=UserTaskStatus.SUCCEEDED, - migrations__is_failed=False, - key__in=source_keys, - ) - .values_list( - 'migrations__target__key', - 'migrations__target__title', - 'migrations__target_collection__key', - 'migrations__target_collection__title', - 'key', - named=True, - ) - } - - -def get_target_block_usage_keys(source_key: CourseKey | LibraryLocator) -> dict[UsageKey, LibraryUsageLocatorV2 | None]: - """ - For given source_key, get a map of legacy block key and its new location in migrated v2 library. - """ - query_set = ModulestoreBlockMigration.objects.filter(overall_migration__source__key=source_key).select_related( - 'source', 'target__component__component_type', 'target__learning_package' - ) - - def construct_usage_key(lib_key_str: str, component: Component) -> LibraryUsageLocatorV2 | None: - try: - lib_key = LibraryLocatorV2.from_string(lib_key_str) - except InvalidKeyError: - return None - return library_component_usage_key(lib_key, component) - - # Use LibraryUsageLocatorV2 and construct usage key - return { - obj.source.key: construct_usage_key(obj.target.learning_package.key, obj.target.component) - for obj in query_set - if obj.source.key is not None - } diff --git a/cms/djangoapps/modulestore_migrator/api/__init__.py b/cms/djangoapps/modulestore_migrator/api/__init__.py new file mode 100644 index 000000000000..f480d92bf092 --- /dev/null +++ b/cms/djangoapps/modulestore_migrator/api/__init__.py @@ -0,0 +1,9 @@ + +""" +This is the public API for the modulestore_migrator. +""" + +# These wildcard imports are okay because these api modules declare __all__. +# pylint: disable=wildcard-import +from .read_api import * +from .write_api import * diff --git a/cms/djangoapps/modulestore_migrator/api/read_api.py b/cms/djangoapps/modulestore_migrator/api/read_api.py new file mode 100644 index 000000000000..893ef416d091 --- /dev/null +++ b/cms/djangoapps/modulestore_migrator/api/read_api.py @@ -0,0 +1,244 @@ +""" +API for reading information about previous migrations +""" +from __future__ import annotations + +import typing as t +from uuid import UUID + +from opaque_keys.edx.keys import UsageKey +from opaque_keys.edx.locator import ( + LibraryLocatorV2, LibraryUsageLocatorV2, LibraryContainerLocator +) +from openedx_learning.api.authoring import get_draft_version +from openedx_learning.api.authoring_models import ( + PublishableEntityVersion, PublishableEntity, DraftChangeLogRecord +) + +from openedx.core.djangoapps.content_libraries.api import ( + library_component_usage_key, library_container_locator +) + +from ..data import ( + SourceContextKey, ModulestoreMigration, ModulestoreBlockMigrationResult, + ModulestoreBlockMigrationSuccess, ModulestoreBlockMigrationFailure +) +from .. import models + + +__all__ = ( + 'get_forwarding', + 'is_forwarded', + 'get_forwarding_for_blocks', + 'get_migrations', + 'get_migration_blocks', +) + + +def get_forwarding_for_blocks(source_keys: t.Iterable[UsageKey]) -> dict[UsageKey, ModulestoreBlockMigrationSuccess]: + """ + Authoritatively determine how some Modulestore blocks have been migrated to Learning Core. + + Returns a mapping from source usage keys to block migration data objects. Each block migration object + holds the target usage key and title. If a source key is missing from the mapping, then it has not + been authoritatively migrated. + """ + sources = models.ModulestoreBlockSource.objects.filter( + key__in=[str(sk) for sk in source_keys] + ).select_related( + "forwarded__target__learning_package", + # For building component key + "forwarded__target__component__component_type", + # For building container key + "forwarded__target__container__section", + "forwarded__target__container__subsection", + "forwarded__target__container__unit", + # For determining title and version + "forwarded__change_log_record__new_version", + ) + result = {} + for source in sources: + if source.forwarded and source.forwarded.target: + result[source.key] = _block_migration_success( + source_key=source.key, + target=source.forwarded.target, + change_log_record=source.forwarded.change_log_record, + ) + return result + + +def is_forwarded(source_key: SourceContextKey) -> bool: + """ + Has this course or legacy library been authoratively migrated to Learning Core, + such that references to the source course/library should be forwarded to the target library? + """ + return get_forwarding(source_key) is not None + + +def get_forwarding(source_key: SourceContextKey) -> ModulestoreMigration | None: + """ + Authoritatively determine how some Modulestore course or legacy library has been migrated to Learning Core. + + If no such successful migration exists, returns None. + + Note: This function may return None for a course or legacy lib that *has* been migrated 1+ times. + This just means that those migrations were non-forwarding. In user parlance, that is, + they have been "imported" but not truly "migrated". + """ + try: + source = models.ModulestoreSource.objects.select_related( + # The following are used in _migration: + "forwarded__source", + "forwarded__target", + "forwarded__task_status", + "forwarded__target_collection", + ).get( + key=str(source_key) + ) + except models.ModulestoreSource.DoesNotExist: + return None + if not source.forwarded: + return None + if source.forwarded.is_failed: + return None + return _migration(source.forwarded) + + +def get_migrations( + source_key: SourceContextKey | None = None, + *, + target_key: LibraryLocatorV2 | None = None, + target_collection_slug: str | None = None, + task_uuid: UUID | None = None, + is_failed: bool | None = None, +) -> t.Generator[ModulestoreMigration]: + """ + Given some criteria, get all modulestore->LearningCore migrations. + + Returns an iterable, ordered from NEWEST to OLDEST. + + Please note: If you provide no filters, this will return an iterable across the whole + ModulestoreMigration table. Please paginate thoughtfully if you do that. + """ + migrations = models.ModulestoreMigration.objects.all().select_related( + "source", + "target", + "target_collection", + "task_status", + ) + if source_key: + migrations = migrations.filter(source__key=source_key) + if target_key: + migrations = migrations.filter(target__key=str(target_key)) + if target_collection_slug: + migrations = migrations.filter(target_collection__key=target_collection_slug) + if task_uuid: + migrations = migrations.filter(task_status__uuid=str(task_uuid)) + if is_failed is not None: + migrations = migrations.filter(is_failed=is_failed) + return ( + _migration(migration) + for migration in migrations.order_by("-id") # primary key is a proxy for newness + ) + + +def get_migration_blocks(migration_pk: int) -> dict[UsageKey, ModulestoreBlockMigrationResult]: + """ + Get details about the migrations of each individual block within a course/lib migration. + """ + return { + block_migration.source.key: _block_migration_result(block_migration) + for block_migration in models.ModulestoreBlockMigration.objects.filter( + overall_migration_id=migration_pk + ).select_related( + "source", + "target__learning_package", + # For building component key + "target__component__component_type", + # For building container key. + # (Hard-coding these exact 3 container types here is not a good pattern, but it's what is needed + # here in order to avoid additional SELECTs while determining the container type). + "target__container__section", + "target__container__subsection", + "target__container__unit", + # For determining title and version + "change_log_record__new_version", + ) + } + + +def _migration(m: models.ModulestoreMigration) -> ModulestoreMigration: + """ + Build a migration dataclass from the database row + """ + return ModulestoreMigration( + pk=m.id, + source_key=m.source.key, + target_key=LibraryLocatorV2.from_string(m.target.key), + target_title=m.target.title, + target_collection_slug=(m.target_collection.key if m.target_collection else None), + target_collection_title=(m.target_collection.title if m.target_collection else None), + is_failed=m.is_failed, + task_uuid=m.task_status.uuid, + ) + + +def _block_migration_result(m: models.ModulestoreBlockMigration) -> ModulestoreBlockMigrationResult: + """ + Build an instance of the migration result (successs/failure) dataclass from a database row + """ + if m.target: + return _block_migration_success( + source_key=m.source.key, + target=m.target, + change_log_record=m.change_log_record, + ) + return ModulestoreBlockMigrationFailure( + source_key=m.source.key, + unsupported_reason=(m.unsupported_reason or ""), + ) + + +def _block_migration_success( + source_key: UsageKey, + target: PublishableEntity, + change_log_record: DraftChangeLogRecord | None, +) -> ModulestoreBlockMigrationSuccess: + """ + Build an instance of the migration success dataclass + """ + target_library_key = LibraryLocatorV2.from_string(target.learning_package.key) + target_key: LibraryUsageLocatorV2 | LibraryContainerLocator + if hasattr(target, "component"): + target_key = library_component_usage_key(target_library_key, target.component) + elif hasattr(target, "container"): + target_key = library_container_locator(target_library_key, target.container) + else: + raise ValueError(f"Entity is neither a container nor component: {target}") + # We expect that any successful BlockMigration (that is, one where `target is not None`) + # will also have a `change_log_record` with a non-None `new_version`. However, the data model + # does not guarantee that `change_log_record` nor `change_log_record.new_version` are non- + # None. So, just in case some bug in the modulestore_migrator or some manual modification of + # the database leads us to a situation where `target` is set but `change_log_record.new_version` + # is not, we have fallback behavior: + # * For target_title, use the latest draft's title, which is good enough, because the + # title is just there to help users. + # * For target_version_num, just use None, because we don't want downstream code to make decisions + # about syncing, etc based on incorrect version info. + target_version: PublishableEntityVersion | None = ( + change_log_record.new_version if change_log_record else None + ) + if target_version: + target_title = target_version.title + target_version_num = target_version.version_num + else: + latest_draft = get_draft_version(target) + target_title = latest_draft.title if latest_draft else "" + target_version_num = None + return ModulestoreBlockMigrationSuccess( + source_key=source_key, + target_entity_pk=target.id, + target_key=target_key, + target_title=target_title, + target_version_num=target_version_num, + ) diff --git a/cms/djangoapps/modulestore_migrator/api/write_api.py b/cms/djangoapps/modulestore_migrator/api/write_api.py new file mode 100644 index 000000000000..4bef6c952767 --- /dev/null +++ b/cms/djangoapps/modulestore_migrator/api/write_api.py @@ -0,0 +1,94 @@ +""" +API for kicking off new migrations +""" +from __future__ import annotations + +from celery.result import AsyncResult +from opaque_keys.edx.locator import LibraryLocatorV2 +from openedx_learning.api.authoring import get_collection + +from openedx.core.types.user import AuthUser +from openedx.core.djangoapps.content_libraries.api import get_library + +from ..data import SourceContextKey, CompositionLevel, RepeatHandlingStrategy +from .. import tasks, models + + +__all__ = ( + 'start_migration_to_library', + 'start_bulk_migration_to_library' +) + + +def start_migration_to_library( + *, + user: AuthUser, + source_key: SourceContextKey, + target_library_key: LibraryLocatorV2, + target_collection_slug: str | None = None, + create_collection: bool = False, + composition_level: CompositionLevel, + repeat_handling_strategy: RepeatHandlingStrategy, + preserve_url_slugs: bool, + forward_source_to_target: bool | None +) -> AsyncResult: + """ + Import a course or legacy library into a V2 library (or, a collection within a V2 library). + """ + return start_bulk_migration_to_library( + user=user, + source_key_list=[source_key], + target_library_key=target_library_key, + target_collection_slug_list=[target_collection_slug], + create_collections=create_collection, + composition_level=composition_level, + repeat_handling_strategy=repeat_handling_strategy, + preserve_url_slugs=preserve_url_slugs, + forward_source_to_target=forward_source_to_target, + ) + + +def start_bulk_migration_to_library( + *, + user: AuthUser, + source_key_list: list[SourceContextKey], + target_library_key: LibraryLocatorV2, + target_collection_slug_list: list[str | None] | None = None, + create_collections: bool = False, + composition_level: CompositionLevel, + repeat_handling_strategy: RepeatHandlingStrategy, + preserve_url_slugs: bool, + forward_source_to_target: bool | None, +) -> AsyncResult: + """ + Import a list of courses or legacy libraries into a V2 library (or, a collections within a V2 library). + """ + target_library = get_library(target_library_key) + # get_library ensures that the library is connected to a learning package. + target_package_id: int = target_library.learning_package_id # type: ignore[assignment] + + sources_pks: list[int] = [] + for source_key in source_key_list: + source, _ = models.ModulestoreSource.objects.get_or_create(key=str(source_key)) + sources_pks.append(source.id) + + target_collection_pks: list[int | None] = [] + if target_collection_slug_list: + for target_collection_slug in target_collection_slug_list: + if target_collection_slug: + target_collection_id = get_collection(target_package_id, target_collection_slug).id + target_collection_pks.append(target_collection_id) + else: + target_collection_pks.append(None) + + return tasks.bulk_migrate_from_modulestore.delay( + user_id=user.id, + sources_pks=sources_pks, + target_library_key=str(target_library_key), + target_collection_pks=target_collection_pks, + create_collections=create_collections, + composition_level=composition_level.value, + repeat_handling_strategy=repeat_handling_strategy.value, + preserve_url_slugs=preserve_url_slugs, + forward_source_to_target=forward_source_to_target, + ) diff --git a/cms/djangoapps/modulestore_migrator/data.py b/cms/djangoapps/modulestore_migrator/data.py index e42649557d67..529d4c78ad40 100644 --- a/cms/djangoapps/modulestore_migrator/data.py +++ b/cms/djangoapps/modulestore_migrator/data.py @@ -1,9 +1,23 @@ """ Value objects """ + from __future__ import annotations +import typing as t +from dataclasses import dataclass from enum import Enum +from uuid import UUID + +from django.utils.translation import gettext_lazy as _ +from opaque_keys.edx.keys import UsageKey +from opaque_keys.edx.locator import ( + CourseLocator, + LibraryContainerLocator, + LibraryLocator, + LibraryLocatorV2, + LibraryUsageLocatorV2, +) from openedx.core.djangoapps.content_libraries.api import ContainerType @@ -70,3 +84,51 @@ def default(cls) -> RepeatHandlingStrategy: Returns the default repeat handling strategy. """ return cls.Skip + + +SourceContextKey: t.TypeAlias = CourseLocator | LibraryLocator + + +@dataclass(frozen=True) +class ModulestoreMigration: + """ + Metadata on a migration of a course or legacy library to a v2 library in learning core. + """ + pk: int + source_key: SourceContextKey + target_key: LibraryLocatorV2 + target_title: str + target_collection_slug: str | None + target_collection_title: str | None + is_failed: bool + task_uuid: UUID # the UserTask which executed this migration + + +@dataclass(frozen=True) +class ModulestoreBlockMigrationResult: + """ + Base class for a modulestore block that was part of an attempted migration to learning core. + """ + source_key: UsageKey + is_failed: t.ClassVar[bool] + + +@dataclass(frozen=True) +class ModulestoreBlockMigrationSuccess(ModulestoreBlockMigrationResult): + """ + Info on a modulestore block which has been successfully migrated into an LC entity + """ + target_entity_pk: int + target_key: LibraryUsageLocatorV2 | LibraryContainerLocator + target_title: str + target_version_num: int | None + is_failed: t.ClassVar[bool] = False + + +@dataclass(frozen=True) +class ModulestoreBlockMigrationFailure(ModulestoreBlockMigrationResult): + """ + Info on a modulestore block which failed to be migrated into LC + """ + unsupported_reason: str + is_failed: t.ClassVar[bool] = True diff --git a/cms/djangoapps/modulestore_migrator/migrations/0004_alter_modulestoreblockmigration_target_squashed_0005_modulestoreblockmigration_unsupported_reason.py b/cms/djangoapps/modulestore_migrator/migrations/0004_alter_modulestoreblockmigration_target_squashed_0005_modulestoreblockmigration_unsupported_reason.py new file mode 100644 index 000000000000..f043e208dc35 --- /dev/null +++ b/cms/djangoapps/modulestore_migrator/migrations/0004_alter_modulestoreblockmigration_target_squashed_0005_modulestoreblockmigration_unsupported_reason.py @@ -0,0 +1,32 @@ +# Generated by Django 5.2.7 on 2025-11-26 06:35 + +import django.db.models.deletion +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ('modulestore_migrator', '0003_modulestoremigration_is_failed'), + ('oel_publishing', '0008_alter_draftchangelogrecord_options_and_more'), + ] + + operations = [ + migrations.AlterField( + model_name='modulestoreblockmigration', + name='target', + field=models.ForeignKey( + blank=True, + help_text='The target entity of this block migration, set to null if it fails to migrate', + null=True, + on_delete=django.db.models.deletion.CASCADE, + to='oel_publishing.publishableentity', + ), + ), + migrations.AddField( + model_name='modulestoreblockmigration', + name='unsupported_reason', + field=models.TextField( + blank=True, help_text='Reason if the block is unsupported and target is set to null', null=True + ), + ), + ] diff --git a/cms/djangoapps/modulestore_migrator/migrations/0006_alter_modulestoreblocksource_forwarded_and_more.py b/cms/djangoapps/modulestore_migrator/migrations/0006_alter_modulestoreblocksource_forwarded_and_more.py new file mode 100644 index 000000000000..a8ee5a0eb673 --- /dev/null +++ b/cms/djangoapps/modulestore_migrator/migrations/0006_alter_modulestoreblocksource_forwarded_and_more.py @@ -0,0 +1,30 @@ +# Generated by Django 5.2.9 on 2025-12-14 15:33 + +import django.db.models.deletion +import opaque_keys.edx.django.models +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('modulestore_migrator', '0004_alter_modulestoreblockmigration_target_squashed_0005_modulestoreblockmigration_unsupported_reason'), + ] + + operations = [ + migrations.AlterField( + model_name='modulestoreblocksource', + name='forwarded', + field=models.OneToOneField(help_text='If set, the system will forward references of this block source over to the target of this block migration', null=True, on_delete=django.db.models.deletion.SET_NULL, to='modulestore_migrator.modulestoreblockmigration'), + ), + migrations.AlterField( + model_name='modulestoreblocksource', + name='key', + field=opaque_keys.edx.django.models.UsageKeyField(help_text='Original usage key of the XBlock that has been imported.', max_length=255, unique=True), + ), + migrations.AlterField( + model_name='modulestoresource', + name='forwarded', + field=models.OneToOneField(blank=True, help_text='If set, the system will forward references of this source over to the target of this migration', null=True, on_delete=django.db.models.deletion.SET_NULL, to='modulestore_migrator.modulestoremigration'), + ), + ] diff --git a/cms/djangoapps/modulestore_migrator/models.py b/cms/djangoapps/modulestore_migrator/models.py index 810333fc9be9..11f614c2ed03 100644 --- a/cms/djangoapps/modulestore_migrator/models.py +++ b/cms/djangoapps/modulestore_migrator/models.py @@ -6,16 +6,19 @@ from django.contrib.auth import get_user_model from django.db import models from django.utils.translation import gettext_lazy as _ -from user_tasks.models import UserTaskStatus - from model_utils.models import TimeStampedModel from opaque_keys.edx.django.models import ( LearningContextKeyField, UsageKeyField, ) from openedx_learning.api.authoring_models import ( - LearningPackage, PublishableEntity, Collection, DraftChangeLog, DraftChangeLogRecord + Collection, + DraftChangeLog, + DraftChangeLogRecord, + LearningPackage, + PublishableEntity, ) +from user_tasks.models import UserTaskStatus from .data import CompositionLevel, RepeatHandlingStrategy @@ -25,6 +28,23 @@ class ModulestoreSource(models.Model): """ A legacy learning context (course or library) which can be a source of a migration. + + One source can be associated with multiple (successful or unsuccessful) ModulestoreMigrations. + If a source has been migrated multiple times, then at most one of them can be considered the + "official" or "authoritative" migration; this is indicated by setting the `forwarded` field to + that ModulestoreMigration object. + + Note that `forwarded` can be NULL even when 1+ migrations have happened for this source. This just + means that none of them were authoritative. In other words, they were all "imports"/"copies" rather + than true "migrations". + + In practice, as of Ulmo: + * The `forwarded` field is used to decide how to update legacy library_content references. + * When using the Libraries Migration UI in Studio, `forwarded` is always set to the first + successful ModulestoreMigration. + * When using the REST API directly, the default is to use the same behavior as the UI, but + clients can also explicitly specify the `forward_source_to_target` boolean param in order to + control whether `forwarded` is set to any given migration. """ key = LearningContextKeyField( max_length=255, @@ -37,7 +57,6 @@ class ModulestoreSource(models.Model): blank=True, on_delete=models.SET_NULL, help_text=_('If set, the system will forward references of this source over to the target of this migration'), - related_name="forwards", ) def __str__(self): @@ -160,6 +179,9 @@ def __repr__(self): class ModulestoreBlockSource(TimeStampedModel): """ A legacy block usage (in a course or library) which can be a source of a block migration. + + The semantics of `forwarded` directly mirror those of `ModulestoreSource.forwarded`. Please see + that class's docstring for details. """ overall_source = models.ForeignKey( ModulestoreSource, @@ -168,6 +190,7 @@ class ModulestoreBlockSource(TimeStampedModel): ) key = UsageKeyField( max_length=255, + unique=True, help_text=_('Original usage key of the XBlock that has been imported.'), ) forwarded = models.OneToOneField( @@ -175,11 +198,10 @@ class ModulestoreBlockSource(TimeStampedModel): null=True, on_delete=models.SET_NULL, help_text=_( - 'If set, the system will forward references of this block source over to the target of this block migration' + 'If set, the system will forward references of this block source over to the ' + 'target of this block migration' ), - related_name="forwards", ) - unique_together = [("overall_source", "key")] def __str__(self): return f"{self.__class__.__name__}('{self.key}')" @@ -210,6 +232,9 @@ class ModulestoreBlockMigration(TimeStampedModel): target = models.ForeignKey( PublishableEntity, on_delete=models.CASCADE, + help_text=_('The target entity of this block migration, set to null if it fails to migrate'), + null=True, + blank=True, ) change_log_record = models.OneToOneField( DraftChangeLogRecord, @@ -218,10 +243,16 @@ class ModulestoreBlockMigration(TimeStampedModel): null=True, on_delete=models.SET_NULL, ) + unsupported_reason = models.TextField( + null=True, + blank=True, + help_text=_('Reason if the block is unsupported and target is set to null'), + ) class Meta: unique_together = [ ('overall_migration', 'source'), + # By default defining a unique index on a nullable column will only enforce unicity of non-null values. ('overall_migration', 'target'), ] diff --git a/cms/djangoapps/modulestore_migrator/rest_api/v1/serializers.py b/cms/djangoapps/modulestore_migrator/rest_api/v1/serializers.py index 73180791191f..643d94d2250c 100644 --- a/cms/djangoapps/modulestore_migrator/rest_api/v1/serializers.py +++ b/cms/djangoapps/modulestore_migrator/rest_api/v1/serializers.py @@ -5,11 +5,25 @@ from opaque_keys import InvalidKeyError from opaque_keys.edx.keys import LearningContextKey from opaque_keys.edx.locator import LibraryLocatorV2 +from openedx_learning.api.authoring_models import Collection from rest_framework import serializers +from user_tasks.models import UserTaskStatus from user_tasks.serializers import StatusSerializer from cms.djangoapps.modulestore_migrator.data import CompositionLevel, RepeatHandlingStrategy -from cms.djangoapps.modulestore_migrator.models import ModulestoreMigration +from cms.djangoapps.modulestore_migrator.models import ( + ModulestoreMigration, + ModulestoreSource, +) + + +class LibraryMigrationCollectionSerializer(serializers.ModelSerializer): + """ + Serializer for the target collection of a library migration. + """ + class Meta: + model = Collection + fields = ["key", "title"] class ModulestoreMigrationSerializer(serializers.Serializer): @@ -40,7 +54,7 @@ class ModulestoreMigrationSerializer(serializers.Serializer): preserve_url_slugs = serializers.BooleanField( help_text="If true, current slugs will be preserved.", required=False, - default=True, + default=False, ) target_collection_slug = serializers.CharField( help_text="The target collection slug within the library to import into. Optional.", @@ -48,10 +62,20 @@ class ModulestoreMigrationSerializer(serializers.Serializer): allow_blank=True, default=None, ) + create_collection = serializers.BooleanField( + help_text=( + "If true and `target_collection_slug` is not set, " + "create the collections in the library where the import will be made" + ), + required=False, + default=False, + ) + target_collection = LibraryMigrationCollectionSerializer(required=False) forward_source_to_target = serializers.BooleanField( help_text="Forward references of this block source over to the target of this block migration.", required=False, - default=False, + allow_null=True, + default=None, # Note: "None" means "unspecified" ) is_failed = serializers.BooleanField( help_text="It is true if this migration is failed", @@ -173,3 +197,95 @@ def get_fields(self): fields = super().get_fields() fields.pop('name', None) return fields + + +class MigrationInfoSerializer(serializers.Serializer): + """ + Serializer for the migration info + """ + + source_key = serializers.CharField() + target_key = serializers.CharField() + target_title = serializers.CharField() + target_collection_key = serializers.CharField( + source="target_collection_slug", + allow_null=True + ) + target_collection_title = serializers.CharField( + allow_null=True + ) + + +class MigrationInfoResponseSerializer(serializers.Serializer): + """ + Serializer for the migrations info view response + """ + def to_representation(self, instance): + return { + str(key): MigrationInfoSerializer(value, many=True).data + for key, value in instance.items() + } + + +class LibraryMigrationCourseSourceSerializer(serializers.ModelSerializer): + """ + Serializer for the source course of a library migration. + """ + display_name = serializers.SerializerMethodField() + + class Meta: + model = ModulestoreSource + fields = ['key', 'display_name'] + + def get_display_name(self, obj): + """ + Return the display name of the source course + """ + return self.context["course_names"].get(str(obj.key), None) + + +class LibraryMigrationCourseSerializer(serializers.ModelSerializer): + """ + Serializer for the course or legacylibrary migrations to V2 library. + """ + task_uuid = serializers.UUIDField(source='task_status.uuid', read_only=True) + source = LibraryMigrationCourseSourceSerializer() # type: ignore[assignment] + target_collection = LibraryMigrationCollectionSerializer(required=False) + state = serializers.SerializerMethodField() + progress = serializers.SerializerMethodField() + + class Meta: + model = ModulestoreMigration + fields = [ + 'task_uuid', + 'source', + 'target_collection', + 'state', + 'progress', + ] + + def get_state(self, obj: ModulestoreMigration): + """ + Return the state of the migration. + """ + if obj.is_failed or obj.task_status.state in [UserTaskStatus.FAILED, UserTaskStatus.CANCELED]: + return UserTaskStatus.FAILED + elif obj.task_status.state == UserTaskStatus.SUCCEEDED: + return UserTaskStatus.SUCCEEDED + + return UserTaskStatus.IN_PROGRESS + + def get_progress(self, obj: ModulestoreMigration): + """ + Return the progress of the migration. + """ + return obj.task_status.completed_steps / obj.task_status.total_steps + + +class BlockMigrationInfoSerializer(serializers.Serializer): + """ + Serializer for the block migration info. + """ + source_key = serializers.CharField() + target_key = serializers.CharField(allow_null=True) + unsupported_reason = serializers.CharField(allow_null=True) diff --git a/cms/djangoapps/modulestore_migrator/rest_api/v1/urls.py b/cms/djangoapps/modulestore_migrator/rest_api/v1/urls.py index 7f66dc5f6dd6..4d2c92b6af04 100644 --- a/cms/djangoapps/modulestore_migrator/rest_api/v1/urls.py +++ b/cms/djangoapps/modulestore_migrator/rest_api/v1/urls.py @@ -1,9 +1,12 @@ """ Course to Library Import API v1 URLs. """ - from rest_framework.routers import SimpleRouter -from .views import MigrationViewSet, BulkMigrationViewSet + +from .views import ( + BulkMigrationViewSet, + MigrationViewSet, +) ROUTER = SimpleRouter() ROUTER.register(r'migrations', MigrationViewSet, basename='migrations') diff --git a/cms/djangoapps/modulestore_migrator/rest_api/v1/views.py b/cms/djangoapps/modulestore_migrator/rest_api/v1/views.py index f2b231c5c1b2..f84f87a23c85 100644 --- a/cms/djangoapps/modulestore_migrator/rest_api/v1/views.py +++ b/cms/djangoapps/modulestore_migrator/rest_api/v1/views.py @@ -6,22 +6,27 @@ import edx_api_doc_tools as apidocs from edx_rest_framework_extensions.auth.jwt.authentication import JwtAuthentication from edx_rest_framework_extensions.auth.session.authentication import SessionAuthenticationAllowInactiveUser -from rest_framework.permissions import IsAdminUser -from rest_framework.response import Response from rest_framework import status +from rest_framework.decorators import action +from rest_framework.exceptions import PermissionDenied +from rest_framework.response import Response from user_tasks.models import UserTaskStatus from user_tasks.views import StatusViewSet -from cms.djangoapps.modulestore_migrator.api import start_migration_to_library, start_bulk_migration_to_library +from cms.djangoapps.modulestore_migrator import api as migrator_api +from common.djangoapps.student import auth +from openedx.core.djangoapps.content_libraries import api as lib_api from openedx.core.lib.api.authentication import BearerAuthenticationAllowInactiveUser +from ...data import ( + CompositionLevel, RepeatHandlingStrategy, +) from .serializers import ( - StatusWithModulestoreMigrationsSerializer, - ModulestoreMigrationSerializer, BulkModulestoreMigrationSerializer, + ModulestoreMigrationSerializer, + StatusWithModulestoreMigrationsSerializer, ) - log = logging.getLogger(__name__) @@ -51,21 +56,11 @@ See `POST /api/modulestore_migrator/v1/migrations` for details on its schema. """, ) -@apidocs.schema_for( - "cancel", - """ - Cancel a particular migration or bulk-migration task. - - The response is a migration task status object. - See `POST /api/modulestore_migrator/v1/migrations` for details on its schema. - """, -) class MigrationViewSet(StatusViewSet): """ JSON HTTP API to create and check on ModuleStore-to-Learning-Core migration tasks. """ - permission_classes = (IsAdminUser,) authentication_classes = ( BearerAuthenticationAllowInactiveUser, JwtAuthentication, @@ -77,14 +72,39 @@ class MigrationViewSet(StatusViewSet): # Instead, users can POST to /cancel to cancel running tasks. http_method_names = ["get", "post"] + lookup_field = "uuid" + def get_queryset(self): """ Override the default queryset to filter by the migration event and user. """ return StatusViewSet.queryset.filter( - migrations__isnull=False, user=self.request.user + migrations__isnull=False, + # The filter for `user` here is essentially the auth strategy for the /list and /retreive + # endpoints. Basically: you can view migrations if and only if you started them. + # Future devs: If you ever refactor this view to remove the user filter, be sure to enforce + # permissions some other way. + user=self.request.user ).distinct().order_by("-created") + @apidocs.schema() + @action(detail=True, methods=['post']) + def cancel(self, request, *args, **kwargs): + """ + Cancel a particular migration or bulk-migration task. + + The response is a migration task status object. + See `POST /api/modulestore_migrator/v1/migrations` for details on its schema. + + This endpoint is currently reserved for site-wide administrators. + """ + # TODO: This should check some sort of "allowed to cancel/migrations" permission + # rather than directly looking at the GlobalStaff role. + # https://github.com/openedx/edx-platform/issues/37791 + if not request.user.is_staff: + raise PermissionDenied("Only site administrators can cancel migration tasks.") + return super().cancel(request, *args, **kwargs) + @apidocs.schema( body=ModulestoreMigrationSerializer, responses={ @@ -94,28 +114,29 @@ def get_queryset(self): ) def create(self, request, *args, **kwargs): """ - Transfer content from course or legacy library into a content library. - - This begins a migration task to copy content from a ModuleStore-based **source** context - to Learning Core-based **target** context. The valid **source** contexts are: - - * A course. - * A legacy content library. - - The valid **target** contexts are: + Begin a transfer of content from course or legacy library into a content library. - * A content library. - * A collection within a content library. + Required parameters: + * A **source** key, which identifies the course or legacy library containing the items be migrated. + * A **target** key, which identifies the content library to which items will be migrated. - Other options: + Optional parameters: + * The **target_collection_slug**, which identifies an *existing* collection within the target that + should hold the migrated items. If not specified, items will be added to the target library without + any collection, unless: + * If this source was previously migrated to a collection and the **repeat_handling_strategy** (described + below) is not set to *fork*, then that same collection will be re-used. + * If **create_collection** is specified as *true*, then the items will be added to a new collection, with + a name and slug based on the source's title, but not conflicting with any existing collection. * The **composition_level** (*component*, *unit*, *subsection*, *section*) indicates the highest level of hierarchy to be transferred. Default is *component*. To maximally preserve the source structure, specify *section*. * The **repeat_handling_strategy** specifies how the system should handle source items which have - previously been migrated to the target. Specify *skip* to prefer the existing target item, specify - *update* to update the existing target item with the latest source content, or specify *fork* to create - a new target item with the source content. Default is *skip*. + previously been migrated to the target. + * Specify *skip* to prefer the existing target item. This is the default. + * Specify *update* to update the existing target item with the latest source content. + * Specify *fork* to create a new target item with the source content. * Specify **preserve_url_slugs** as *true* in order to use the source-provided block IDs (a.k.a. "URL slugs", "url_names"). Otherwise, the system will use each source item's title to auto-generate an ID in the target context. @@ -130,12 +151,17 @@ def create(self, request, *args, **kwargs): content library. * **Example**: Specify *false* if you are copying course content into a content library, but do not want to persist a link between the source source content and destination library contenet. + * **Defaults** to *false* if the source has already been mapped to a target by a successful migration, + and defaults to *true* if not. In other words, by default, establish the mapping only if it wouldn't + override an existing mapping. Example request: ```json { "source": "course-v1:MyOrganization+MyCourse+MyRun", - "target": "lib:MyOrganization:MyUlmoLibrary", + "target": "lib-collection:MyOrganization:MyUlmoLibrary", + "target_collection_slug": "MyCollection", + "create_collection": true, "composition_level": "unit", "repeat_handling_strategy": "update", "preserve_url_slugs": true @@ -183,25 +209,32 @@ def create(self, request, *args, **kwargs): ] } ``` + + This API requires that the requester have author access on both the source and target. """ serializer_data = ModulestoreMigrationSerializer(data=request.data) serializer_data.is_valid(raise_exception=True) validated_data = serializer_data.validated_data - - task = start_migration_to_library( + if not auth.has_studio_write_access(request.user, validated_data['source']): + raise PermissionDenied("Requester is not an author on the source.") + lib_api.require_permission_for_library_key( + validated_data['target'], + request.user, + lib_api.permissions.CAN_EDIT_THIS_CONTENT_LIBRARY, + ) + task = migrator_api.start_migration_to_library( user=request.user, source_key=validated_data['source'], target_library_key=validated_data['target'], target_collection_slug=validated_data['target_collection_slug'], - composition_level=validated_data['composition_level'], - repeat_handling_strategy=validated_data['repeat_handling_strategy'], + composition_level=CompositionLevel(validated_data['composition_level']), + create_collection=validated_data['create_collection'], + repeat_handling_strategy=RepeatHandlingStrategy(validated_data['repeat_handling_strategy']), preserve_url_slugs=validated_data['preserve_url_slugs'], forward_source_to_target=validated_data['forward_source_to_target'], ) - task_status = UserTaskStatus.objects.get(task_id=task.id) serializer = self.get_serializer(task_status) - return Response(serializer.data, status=status.HTTP_201_CREATED) @@ -210,7 +243,6 @@ class BulkMigrationViewSet(StatusViewSet): JSON HTTP API to bulk-create ModuleStore-to-Learning-Core migration tasks. """ - permission_classes = (IsAdminUser,) authentication_classes = ( BearerAuthenticationAllowInactiveUser, JwtAuthentication, @@ -295,19 +327,30 @@ def create(self, request, *args, **kwargs): ] } ``` + + This API requires that the requester have author access on both the source and target. """ serializer_data = BulkModulestoreMigrationSerializer(data=request.data) serializer_data.is_valid(raise_exception=True) validated_data = serializer_data.validated_data - - task = start_bulk_migration_to_library( + for source_key in validated_data['sources']: + if not auth.has_studio_write_access(request.user, source_key): + raise PermissionDenied( + f"Requester is not an author on the source: {source_key}. No migrations performed." + ) + lib_api.require_permission_for_library_key( + validated_data['target'], + request.user, + lib_api.permissions.CAN_EDIT_THIS_CONTENT_LIBRARY, + ) + task = migrator_api.start_bulk_migration_to_library( user=request.user, source_key_list=validated_data['sources'], target_library_key=validated_data['target'], target_collection_slug_list=validated_data['target_collection_slug_list'], create_collections=validated_data['create_collections'], - composition_level=validated_data['composition_level'], - repeat_handling_strategy=validated_data['repeat_handling_strategy'], + composition_level=CompositionLevel(validated_data['composition_level']), + repeat_handling_strategy=RepeatHandlingStrategy(validated_data['repeat_handling_strategy']), preserve_url_slugs=validated_data['preserve_url_slugs'], forward_source_to_target=validated_data['forward_source_to_target'], ) diff --git a/cms/djangoapps/modulestore_migrator/tasks.py b/cms/djangoapps/modulestore_migrator/tasks.py index ef2386c69d34..e0b0b0fe2ed2 100644 --- a/cms/djangoapps/modulestore_migrator/tasks.py +++ b/cms/djangoapps/modulestore_migrator/tasks.py @@ -9,24 +9,26 @@ from dataclasses import dataclass from datetime import datetime, timezone from enum import Enum -from itertools import groupby +from gettext import ngettext from celery import shared_task from celery.utils.log import get_task_logger from django.core.exceptions import ObjectDoesNotExist -from django.utils.text import slugify from django.db import transaction +from django.utils.text import slugify +from django.utils.translation import gettext_lazy as _ from edx_django_utils.monitoring import set_code_owner_attribute_from_module from lxml import etree from lxml.etree import _ElementTree as XmlTree from opaque_keys import InvalidKeyError -from opaque_keys.edx.keys import CourseKey, UsageKey +from opaque_keys.edx.keys import UsageKey from opaque_keys.edx.locator import ( + BlockUsageLocator, CourseLocator, LibraryContainerLocator, LibraryLocator, LibraryLocatorV2, - LibraryUsageLocatorV2 + LibraryUsageLocatorV2, ) from openedx_learning.api import authoring as authoring_api from openedx_learning.api.authoring_models import ( @@ -35,23 +37,24 @@ ComponentType, LearningPackage, PublishableEntity, - PublishableEntityVersion + PublishableEntityVersion, ) from user_tasks.tasks import UserTask, UserTaskStatus from xblock.core import XBlock -from django.utils.translation import gettext_lazy as _ +from xblock.plugin import PluginMissingError from common.djangoapps.split_modulestore_django.models import SplitModulestoreCourseIndex -from common.djangoapps.util.date_utils import strftime_localized, DEFAULT_DATE_TIME_FORMAT +from common.djangoapps.util.date_utils import DEFAULT_DATE_TIME_FORMAT, strftime_localized from openedx.core.djangoapps.content_libraries import api as libraries_api from openedx.core.djangoapps.content_libraries.api import ContainerType, get_library from openedx.core.djangoapps.content_staging import api as staging_api from xmodule.modulestore import exceptions as modulestore_exceptions from xmodule.modulestore.django import modulestore +from . import models, data from .constants import CONTENT_STAGING_PURPOSE_TEMPLATE -from .data import CompositionLevel, RepeatHandlingStrategy -from .models import ModulestoreBlockMigration, ModulestoreBlockSource, ModulestoreMigration, ModulestoreSource +from .data import CompositionLevel, RepeatHandlingStrategy, SourceContextKey +from .api.read_api import get_migrations, get_migration_blocks log = get_task_logger(__name__) @@ -78,20 +81,6 @@ class MigrationStep(Enum): BULK_MIGRATION_PREFIX = 'Migrating legacy content' -class _MigrationTask(UserTask): - """ - Base class for migrate_to_modulestore - """ - - @staticmethod - def calculate_total_steps(arguments_dict): - """ - Get number of in-progress steps in importing process, as shown in the UI. - """ - # We subtract the BULK_MIGRATION_PREFIX - return len(list(MigrationStep)) - 1 - - class _BulkMigrationTask(UserTask): """ Base class for bulk_migrate_from_modulestore @@ -125,12 +114,15 @@ class _MigrationContext: """ Context for the migration process. """ - existing_source_to_target_keys: dict[ # Note: It's intended to be mutable to reflect changes during migration. - UsageKey, list[PublishableEntity] - ] + # Fields that get mutated as we migrate blocks + used_component_keys: set[LibraryUsageLocatorV2] + used_container_slugs: set[str] + + # Fields that remain constant + previous_block_migrations: dict[UsageKey, data.ModulestoreBlockMigrationResult] target_package_id: int target_library_key: LibraryLocatorV2 - source_context_key: CourseKey # Note: This includes legacy LibraryLocators, which are sneakily CourseKeys. + source_context_key: SourceContextKey content_by_filename: dict[str, int] composition_level: CompositionLevel repeat_handling_strategy: RepeatHandlingStrategy @@ -138,37 +130,6 @@ class _MigrationContext: created_by: int created_at: datetime - def is_already_migrated(self, source_key: UsageKey) -> bool: - return source_key in self.existing_source_to_target_keys - - def get_existing_target(self, source_key: UsageKey) -> PublishableEntity: - """ - Get the target entity for a given source key. - - If the source key is already migrated, return the FIRST target entity. - If the source key is not found, raise a KeyError. - """ - if source_key not in self.existing_source_to_target_keys: - raise KeyError(f"Source key {source_key} not found in existing source to target keys") - - # NOTE: This is a list of PublishableEntities, but we always return the first one. - return self.existing_source_to_target_keys[source_key][0] - - def add_migration(self, source_key: UsageKey, target: PublishableEntity) -> None: - """Update the context with a new migration (keeps it current)""" - if source_key not in self.existing_source_to_target_keys: - self.existing_source_to_target_keys[source_key] = [target] - else: - self.existing_source_to_target_keys[source_key].append(target) - - def get_existing_target_entity_keys(self, base_key: str) -> set[str]: - return set( - publishable_entity.key - for publishable_entity_list in self.existing_source_to_target_keys.values() - for publishable_entity in publishable_entity_list - if publishable_entity.key.startswith(base_key) - ) - @property def should_skip_strategy(self) -> bool: """ @@ -196,10 +157,11 @@ class _MigrationSourceData: """ Data related to a ModulestoreSource """ - source: ModulestoreSource - source_root_usage_key: UsageKey + source: models.ModulestoreSource + source_root_usage_key: BlockUsageLocator source_version: str | None - migration: ModulestoreMigration + migration: models.ModulestoreMigration + previous_migration: data.ModulestoreMigration | None def _validate_input( @@ -208,6 +170,7 @@ def _validate_input( repeat_handling_strategy: str, preserve_url_slugs: bool, composition_level: str, + target_library_key: LibraryLocatorV2, target_package: LearningPackage, target_collection: Collection | None, ) -> _MigrationSourceData | None: @@ -215,7 +178,7 @@ def _validate_input( Validates and build the source data related to `source_pk` """ try: - source = ModulestoreSource.objects.get(pk=source_pk) + source = models.ModulestoreSource.objects.get(pk=source_pk) except (ObjectDoesNotExist) as exc: status.fail(str(exc)) return None @@ -235,7 +198,17 @@ def _validate_input( ) return None - migration = ModulestoreMigration.objects.create( + # Find the latest successful migration that occurred, if any. + # We're careful to do this before creating the new ModulestoreMigration object, + # otherwise we would just end up grabbing that by one accident. + # ( mypy gets confused by how use next(...) here ) + previous_migration = next( # type: ignore[call-overload] + get_migrations( + source.key, target_key=target_library_key, is_failed=False + ), + None, # default + ) + migration = models.ModulestoreMigration.objects.create( source=source, source_version=source_version, composition_level=composition_level, @@ -245,17 +218,17 @@ def _validate_input( target_collection=target_collection, task_status=status, ) - return _MigrationSourceData( source=source, source_root_usage_key=source_root_usage_key, source_version=source_version, migration=migration, + previous_migration=previous_migration, ) def _cancel_old_tasks( - source_list: list[ModulestoreSource], + source_list: list[models.ModulestoreSource], status: UserTaskStatus, target_package: LearningPackage, migration_ids_to_exclude: list[int], @@ -264,7 +237,7 @@ def _cancel_old_tasks( Cancel all migration tasks related to the user and the source list """ # In order to prevent a user from accidentally starting a bunch of identical import tasks... - migrations_to_cancel = ModulestoreMigration.objects.filter( + migrations_to_cancel = models.ModulestoreMigration.objects.filter( # get all Migration tasks by this user with the same sources and target task_status__user=status.user, source__in=source_list, @@ -302,7 +275,7 @@ def _load_xblock( return xblock -def _import_assets(migration: ModulestoreMigration) -> dict[str, int]: +def _import_assets(migration: models.ModulestoreMigration) -> dict[str, int]: """ Import the assets of the staged content to the migration target """ @@ -333,7 +306,6 @@ def _import_assets(migration: ModulestoreMigration) -> dict[str, int]: def _import_structure( - migration: ModulestoreMigration, source_data: _MigrationSourceData, target_library: libraries_api.ContentLibraryMetadata, content_by_filename: dict[str, int], @@ -368,21 +340,25 @@ def _import_structure( represents the mapping between the legacy root node and its newly created Learning Core equivalent. """ - # "key" is locally unique across all PublishableEntities within - # a given LearningPackage. - # We use this mapping to ensure that we don't create duplicate - # PublishableEntities during the migration process for a given LearningPackage. - existing_source_to_target_keys: dict[UsageKey, list[PublishableEntity]] = {} - modulestore_blocks = ( - ModulestoreBlockMigration.objects.filter(overall_migration__target=migration.target.id).order_by("source__key") - ) - existing_source_to_target_keys = { - source_key: list(block.target for block in group) for source_key, group in groupby( - modulestore_blocks, key=lambda x: x.source.key) - } - + migration = source_data.migration migration_context = _MigrationContext( - existing_source_to_target_keys=existing_source_to_target_keys, + used_component_keys=set( + LibraryUsageLocatorV2(target_library.key, block_type, block_id) # type: ignore[abstract] + for block_type, block_id + in authoring_api.get_components(migration.target.pk).values_list( + "component_type__name", "local_key" + ) + ), + used_container_slugs=set( + authoring_api.get_containers( + migration.target.pk + ).values_list("publishable_entity__key", flat=True) + ), + previous_block_migrations=( + get_migration_blocks(source_data.previous_migration.pk) + if source_data.previous_migration + else {} + ), target_package_id=migration.target.pk, target_library_key=target_library.key, source_context_key=source_data.source_root_usage_key.course_key, @@ -393,7 +369,6 @@ def _import_structure( created_by=status.user_id, created_at=datetime.now(timezone.utc), ) - with authoring_api.bulk_draft_changes_for(migration.target.id) as change_log: root_migrated_node = _migrate_node( context=migration_context, @@ -403,11 +378,11 @@ def _import_structure( return change_log, root_migrated_node -def _forwarding_content(source_data: _MigrationSourceData) -> None: +def _forward_content(source_data: _MigrationSourceData) -> None: """ Forwarding legacy content to migrated content """ - block_migrations = ModulestoreBlockMigration.objects.filter(overall_migration=source_data.migration) + block_migrations = models.ModulestoreBlockMigration.objects.filter(overall_migration=source_data.migration) block_sources_to_block_migrations = { block_migration.source: block_migration for block_migration in block_migrations } @@ -419,7 +394,7 @@ def _forwarding_content(source_data: _MigrationSourceData) -> None: source_data.source.save() -def _populate_collection(user_id: int, migration: ModulestoreMigration) -> None: +def _populate_collection(user_id: int, migration: models.ModulestoreMigration) -> None: """ Assigning imported items to the specified collection in the migration """ @@ -427,7 +402,7 @@ def _populate_collection(user_id: int, migration: ModulestoreMigration) -> None: return block_target_pks: list[int] = list( - ModulestoreBlockMigration.objects.filter( + models.ModulestoreBlockMigration.objects.filter( overall_migration=migration ).values_list('target_id', flat=True) ) @@ -450,7 +425,7 @@ def _create_collection(library_key: LibraryLocatorV2, title: str) -> Collection: The same is true for the title. """ key = slugify(title) - collection = None + collection: Collection | None = None attempt = 0 created_at = strftime_localized(datetime.now(timezone.utc), DEFAULT_DATE_TIME_FORMAT) description = f"{_('This collection contains content migrated from a legacy library on')}: {created_at}" @@ -465,7 +440,7 @@ def _create_collection(library_key: LibraryLocatorV2, title: str) -> Collection: title=f"{title}{f'_{attempt}' if attempt > 0 else ''}", description=description, ) - except libraries_api.LibraryCollectionAlreadyExists as e: + except libraries_api.LibraryCollectionAlreadyExists: attempt += 1 return collection @@ -477,186 +452,12 @@ def _set_migrations_to_fail(source_data_list: list[_MigrationSourceData]): for source_data in source_data_list: source_data.migration.is_failed = True - ModulestoreMigration.objects.bulk_update( + models.ModulestoreMigration.objects.bulk_update( [x.migration for x in source_data_list], ["is_failed"], ) -@shared_task(base=_MigrationTask, bind=True) -# Note: The decorator @set_code_owner_attribute cannot be used here because the UserTaskMixin -# does stack inspection and can't handle additional decorators. -def migrate_from_modulestore( - self: _MigrationTask, - *, - user_id: int, - source_pk: int, - target_library_key: str, - target_collection_pk: int | None, - repeat_handling_strategy: str, - preserve_url_slugs: bool, - composition_level: str, - forward_source_to_target: bool, -) -> None: - """ - Import a single course or legacy library from modulestore into a V2 legacy library. - - This task performs the end-to-end migration for one legacy source (course or library), - including staging, parsing OLX, importing assets and structure, and assigning the - migrated content to the specified target library and collection. - - A new `UserTaskStatus` entry is created for each invocation of this task, meaning - that each migration runs independently with its own progress tracking and final - success or failure state. - - If the migration encounters an unrecoverable error at any step (for example, invalid - OLX, missing assets, or database constraints), the task is marked as **failed** and - the partial results are rolled back as necessary. The migration state can be queried - through the REST API endpoint `/api/modulestore_migrator/v1/migrations//`. - - Args: - self (_MigrationTask): - The Celery task instance that wraps the user task logic. - user_id (int): - The ID of the user initiating the migration. - source_pk (int): - Primary key of the modulestore source to migrate. - target_library_key (str): - Key of the target V2 library that will receive the imported content. - target_collection_pk (int | None): - Optional ID of a target collection to which imported content will be assigned. - repeat_handling_strategy (str): - Strategy for handling repeated imports (e.g., "skip", "update"). - preserve_url_slugs (bool): - Whether to preserve original XBlock URL slugs during import. - composition_level (str): - The structural level to migrate (e.g., component, unit, or section). - forward_source_to_target (bool): - Whether to forward legacy content references to the migrated content after import. - - See Also: - - `bulk_migrate_from_modulestore`: Multi-source batch migration equivalent. - - API docs: `/api/cms/v1/migrations/` for REST behavior and responses. - """ - - # pylint: disable=too-many-statements - # This is a large function, but breaking it up futher would probably not - # make it any easier to understand. - - set_code_owner_attribute_from_module(__name__) - status: UserTaskStatus = self.status - - # Validating input - status.set_state(MigrationStep.VALIDATING_INPUT.value) - try: - target_library = get_library(LibraryLocatorV2.from_string(target_library_key)) - if target_library.learning_package_id is None: - raise ValueError("Target library has no associated learning package.") - - target_package = LearningPackage.objects.get(pk=target_library.learning_package_id) - target_collection = Collection.objects.get(pk=target_collection_pk) if target_collection_pk else None - except (ObjectDoesNotExist, InvalidKeyError) as exc: - status.fail(str(exc)) - return - - source_data = _validate_input( - status, - source_pk, - repeat_handling_strategy, - preserve_url_slugs, - composition_level, - target_package, - target_collection, - ) - if source_data is None: - # Fail - return - - migration = source_data.migration - status.increment_completed_steps() - - try: - # Cancelling old tasks - status.set_state(MigrationStep.CANCELLING_OLD.value) - _cancel_old_tasks([source_data.source], status, target_package, [migration.id]) - status.increment_completed_steps() - - # Loading `legacy_root` - status.set_state(MigrationStep.LOADING) - legacy_root = _load_xblock(status, source_data.source_root_usage_key) - if legacy_root is None: - # Fail - _set_migrations_to_fail([source_data]) - return - status.increment_completed_steps() - - # Staging legacy block - status.set_state(MigrationStep.STAGING.value) - staged_content = staging_api.stage_xblock_temporarily( - block=legacy_root, - user_id=status.user.pk, - purpose=CONTENT_STAGING_PURPOSE_TEMPLATE.format(source_key=source_pk), - ) - migration.staged_content = staged_content - status.increment_completed_steps() - - # Parsing OLX - status.set_state(MigrationStep.PARSING.value) - parser = etree.XMLParser(strip_cdata=False) - try: - root_node = etree.fromstring(staged_content.olx, parser=parser) - except etree.ParseError as exc: - status.fail(f"Failed to parse source OLX (from staged content with id = {staged_content.id}): {exc}") - _set_migrations_to_fail([source_data]) - return - status.increment_completed_steps() - - # Importing assets of the legacy block - status.set_state(MigrationStep.IMPORTING_ASSETS.value) - content_by_filename = _import_assets(migration) - status.increment_completed_steps() - - # Importing structure of the legacy block - status.set_state(MigrationStep.IMPORTING_STRUCTURE.value) - change_log, root_migrated_node = _import_structure( - migration, - source_data, - target_library, - content_by_filename, - root_node, - status, - ) - migration.change_log = change_log - status.increment_completed_steps() - - status.set_state(MigrationStep.UNSTAGING.value) - staged_content.delete() - status.increment_completed_steps() - - _create_migration_artifacts_incrementally( - root_migrated_node=root_migrated_node, - source=source_data.source, - migration=migration, - status=status, - ) - status.increment_completed_steps() - - # Forwarding legacy content to migrated content - status.set_state(MigrationStep.FORWARDING.value) - if forward_source_to_target: - _forwarding_content(source_data) - status.increment_completed_steps() - - # Populating the collection - status.set_state(MigrationStep.POPULATING_COLLECTION.value) - if target_collection: - _populate_collection(user_id, migration) - status.increment_completed_steps() - except Exception as exc: # pylint: disable=broad-exception-caught - _set_migrations_to_fail([source_data]) - status.fail(str(exc)) - - @shared_task(base=_BulkMigrationTask, bind=True) # Note: The decorator @set_code_owner_attribute cannot be used here because the UserTaskMixin # does stack inspection and can't handle additional decorators. @@ -671,19 +472,14 @@ def bulk_migrate_from_modulestore( repeat_handling_strategy: str, preserve_url_slugs: bool, composition_level: str, - forward_source_to_target: bool, + forward_source_to_target: bool | None, ) -> None: """ Import multiple legacy courses or libraries into a single V2 library. - This task performs the same logical steps as `migrate_from_modulestore`, but allows - batching several migrations together under **one single user task** (`UserTaskStatus`). - - Unlike running `migrate_from_modulestore` in a loop (which would create multiple - independent Celery tasks and separate statuses), the bulk migration maintains - **one unified status record** that tracks progress across all included sources. - This simplifies monitoring, since the client only needs to observe one task state. - + The bulk migration maintains **one unified status record** that tracks progress across + all included sources. This simplifies monitoring, since the client only needs to observe + one task state. Each source item (course or library) still creates its own `ModulestoreMigration` database record, but all of them share the same parent task (`UserTaskStatus`). If any sub-migration fails (for example, due to invalid OLX or missing assets), @@ -708,8 +504,10 @@ def bulk_migrate_from_modulestore( Whether to preserve existing XBlock URL slugs during import. composition_level (str): Composition level at which content should be imported (e.g. course, section). - forward_source_to_target (bool): + forward_source_to_target (bool | None) Whether to forward legacy content to its migrated equivalent after import. + If unspecified (None), then forward legacy content for a source if and only + if it's that source's first migration. See Also: - `migrate_from_modulestore`: Single-source migration equivalent. @@ -752,13 +550,13 @@ def bulk_migrate_from_modulestore( repeat_handling_strategy, preserve_url_slugs, composition_level, + target_library_locator, target_package, target_collection_list[i] if target_collection_list else None, ) if source_data is None: # Fail return - source_data_list.append(source_data) status.increment_completed_steps() @@ -825,14 +623,14 @@ def bulk_migrate_from_modulestore( f"{MigrationStep.IMPORTING_STRUCTURE.value}" ) change_log, root_migrated_node = _import_structure( - source_data.migration, - source_data, - target_library, - content_by_filename, - root_node, - status, + source_data=source_data, + target_library=target_library, + content_by_filename=content_by_filename, + root_node=root_node, + status=status, ) source_data.migration.change_log = change_log + source_data.migration.save() # @@TODO keep or nah? status.increment_completed_steps() status.set_state( @@ -849,7 +647,8 @@ def bulk_migrate_from_modulestore( source_pk=source_pk, ) status.increment_completed_steps() - except: # pylint: disable=bare-except + except Exception as _exc: # pylint: disable=broad-exception-caught + log.exception("Failed: {source_data.migration}") # Mark this library as failed, migration of other libraries can continue # If this case occurs and the migration ends without any further issues, # the bulk migration status is success, @@ -858,73 +657,60 @@ def bulk_migrate_from_modulestore( # Forwarding legacy content to migrated content status.set_state(MigrationStep.FORWARDING.value) - if forward_source_to_target: - for source_data in source_data_list: - if not source_data.migration.is_failed: - _forwarding_content(source_data) + for source_data in source_data_list: + if forward_source_to_target is False: + continue # Explicitly requested not to forward. + if forward_source_to_target is None and source_data.source.forwarded: + # Unspecified whether or not to forward. + # So, forward iff there was no previous existing successful migration with forwarding. + continue + if source_data.migration.is_failed: + # Don't forward failed migrations. + continue + _forward_content(source_data) status.increment_completed_steps() # Populating collections status.set_state(MigrationStep.POPULATING_COLLECTION.value) - - # Used to check if the source has a previous migration in a V2 library collection - # It is placed here to avoid the circular import - from .api import get_migration_info for i, source_data in enumerate(source_data_list): migration = source_data.migration if migration.is_failed: continue - - title = legacy_root_list[i].display_name + if migration.target_collection is None and not create_collections: + continue if migration.target_collection is None: - if not create_collections: - continue - - source_key = source_data.source.key - - if migration.repeat_handling_strategy == RepeatHandlingStrategy.Fork.value: - # Create a new collection when it is Fork - migration.target_collection = _create_collection(target_library_locator, title) - else: - # It is Skip or Update - # We need to verify if there is a previous migration with collection - # TODO: This only fetches the latest migration, if different migrations have been done - # on different V2 libraries, this could break the logic. - previous_migration = get_migration_info([source_key]) - if ( - source_key in previous_migration - and previous_migration[source_key].migrations__target_collection__key - ): - # Has previous migration with collection - try: - # Get the previous collection - previous_collection = authoring_api.get_collection( - target_package.id, - previous_migration[source_key].migrations__target_collection__key, - ) - - migration.target_collection = previous_collection - except Collection.DoesNotExist: - # The collection no longer exists or is being migrated to a different library. - # In that case, create a new collection independent of strategy - migration.target_collection = _create_collection(target_library_locator, title) - else: - # Create collection and save in migration - migration.target_collection = _create_collection(target_library_locator, title) - + existing_collection_to_use: Collection | None = None + # For Fork strategy: Create an new collection every time. + # For Update and Skip strategies: Update an existing collection if possible. + if migration.repeat_handling_strategy != RepeatHandlingStrategy.Fork.value: + if source_data.previous_migration: + if previous_collection_slug := source_data.previous_migration.target_collection_slug: + try: + existing_collection_to_use = authoring_api.get_collection( + target_package.id, previous_collection_slug + ) + except Collection.DoesNotExist: + # Collection no longer exists. + pass + migration.target_collection = ( + existing_collection_to_use or + _create_collection(library_key=target_library_locator, title=legacy_root_list[i].display_name) + ) _populate_collection(user_id, migration) - - ModulestoreMigration.objects.bulk_update( + models.ModulestoreMigration.objects.bulk_update( [x.migration for x in source_data_list], ["target_collection", "is_failed"], ) status.increment_completed_steps() except Exception as exc: # pylint: disable=broad-exception-caught # If there is an exception in this block, all migrations fail. - _set_migrations_to_fail(source_data_list) + log.exception("Modulestore migrations failed") status.fail(str(exc)) +SourceToTarget = tuple[UsageKey, PublishableEntityVersion | None, str | None] + + @dataclass(frozen=True) class _MigratedNode: """ @@ -934,10 +720,10 @@ class _MigratedNode: This happens, particularly, if the node is above the requested composition level but has descendents which are at or below that level. """ - source_to_target: tuple[UsageKey, PublishableEntityVersion] | None + source_to_target: SourceToTarget | None children: list[_MigratedNode] - def all_source_to_target_pairs(self) -> t.Iterable[tuple[UsageKey, PublishableEntityVersion]]: + def all_source_to_target_pairs(self) -> t.Iterable[SourceToTarget]: """ Get all source_key->target_ver pairs via a pre-order traversal. """ @@ -995,13 +781,13 @@ def _migrate_node( ) for source_node_child in source_node.getchildren() ] - source_to_target: tuple[UsageKey, PublishableEntityVersion] | None = None + source_to_target: SourceToTarget | None = None if should_migrate_node: source_olx = etree.tostring(source_node).decode('utf-8') if source_block_id := source_node.get('url_name'): source_key: UsageKey = context.source_context_key.make_usage_key(source_node.tag, source_block_id) title = source_node.get('display_name', source_block_id) - target_entity_version = ( + target_entity_version, reason = ( _migrate_container( context=context, source_key=source_key, @@ -1010,7 +796,7 @@ def _migrate_node( children=[ migrated_child.source_to_target[1] for migrated_child in migrated_children if - migrated_child.source_to_target + migrated_child.source_to_target and migrated_child.source_to_target[1] ], ) if container_type else @@ -1021,9 +807,18 @@ def _migrate_node( title=title, ) ) - if target_entity_version: - source_to_target = (source_key, target_entity_version) - context.add_migration(source_key, target_entity_version.entity) + if container_type is None and target_entity_version is None and reason is not None: + # Currently, components with children are not supported + children_length = len(source_node.getchildren()) + if children_length: + reason += ( + ngettext( + ' It has {count} children block.', + ' It has {count} children blocks.', + children_length, + ) + ).format(count=children_length) + source_to_target = (source_key, target_entity_version, reason) else: log.warning( f"Cannot migrate node from {context.source_context_key} to {context.target_library_key} " @@ -1039,12 +834,14 @@ def _migrate_container( container_type: ContainerType, title: str, children: list[PublishableEntityVersion], -) -> PublishableEntityVersion: +) -> tuple[PublishableEntityVersion, str | None]: """ Create, update, or replace a container in a library based on a source key and children. (We assume that the destination is a library rather than some other future kind of learning - package, but let's keep than an internal assumption.) + package, but let's keep than an internal assumption.) + For now this returns None value for unsupported_reason as second value of tuple as we + don't have any concrete condition where a container cannot be imported/migrated. """ target_key = _get_distinct_target_container_key( context, @@ -1076,7 +873,7 @@ def _migrate_container( return PublishableEntityVersion.objects.get( entity_id=container.container_pk, version_num=container.draft_version_num, - ) + ), None container_publishable_entity_version = authoring_api.create_next_container_version( container.container_pk, @@ -1099,7 +896,8 @@ def _migrate_container( context.created_by, call_post_publish_events_sync=True, ) - return container_publishable_entity_version + context.used_container_slugs.add(container.container_key.container_id) + return container_publishable_entity_version, None def _migrate_component( @@ -1108,7 +906,7 @@ def _migrate_component( source_key: UsageKey, olx: str, title: str, -) -> PublishableEntityVersion | None: +) -> tuple[PublishableEntityVersion | None, str | None]: """ Create, update, or replace a component in a library based on a source key and OLX. @@ -1141,7 +939,10 @@ def _migrate_component( ) except libraries_api.IncompatibleTypesError as e: log.error(f"Error validating block for library {context.target_library_key}: {e}") - return None + return None, str(e) + except PluginMissingError as e: + log.error(f"Block type not supported in {context.target_library_key}: {e}") + return None, f"Invalid block type: {e}" component = authoring_api.create_component( context.target_package_id, component_type=component_type, @@ -1152,7 +953,7 @@ def _migrate_component( # Component existed and we do not replace it and it is not deleted previously if component_existed and not component_deleted and context.should_skip_strategy: - return component.versioning.draft.publishable_entity_version + return component.versioning.draft.publishable_entity_version, None # If component existed and was deleted or we have to replace the current version # Create the new component version for it @@ -1167,11 +968,12 @@ def _migrate_component( ) # Publish the component - libraries_api.publish_component_changes( - libraries_api.library_component_usage_key(context.target_library_key, component), - context.created_by, - ) - return component_version.publishable_entity_version + libraries_api.publish_component_changes(target_key, context.created_by) + context.used_component_keys.add(target_key) + return component_version.publishable_entity_version, None + + +_MAX_UNIQUE_SLUG_ATTEMPTS = 1000 def _get_distinct_target_container_key( @@ -1181,39 +983,36 @@ def _get_distinct_target_container_key( title: str, ) -> LibraryContainerLocator: """ - Find a unique key for block_id by appending a unique identifier if necessary. - - Args: - context (_MigrationContext): The migration context. - source_key (UsageKey): The source key. - container_type (ContainerType): The container type. - title (str): The title. - - Returns: - LibraryContainerLocator: The target container key. + Figure out the appropriate target container for this structural block. """ - # Check if we already processed this block and we are not forking. If we are forking, we will - # want a new target key. - if context.is_already_migrated(source_key) and not context.should_fork_strategy: - existing_version = context.get_existing_target(source_key) - - return LibraryContainerLocator( - context.target_library_key, - container_type.value, - existing_version.key - ) + # If we're not forking, then check if this block was part of our past migration. + # (If we are forking, we will always want a new target key). + if not context.should_fork_strategy: + if previous_block_migration := context.previous_block_migrations.get(source_key): + if isinstance(previous_block_migration, data.ModulestoreBlockMigrationSuccess): + if isinstance(previous_block_migration.target_key, LibraryContainerLocator): + return previous_block_migration.target_key # Generate new unique block ID base_slug = ( source_key.block_id if context.preserve_url_slugs else (slugify(title) or source_key.block_id) ) - unique_slug = _find_unique_slug(context, base_slug) - - return LibraryContainerLocator( - context.target_library_key, - container_type.value, - unique_slug + # Use base base slug if available + if base_slug not in context.used_container_slugs: + return LibraryContainerLocator( + context.target_library_key, container_type.value, base_slug + ) + # Try numbered variations until we find one that doesn't exist + for i in range(1, _MAX_UNIQUE_SLUG_ATTEMPTS + 1): + candidate_slug = f"{base_slug}_{i}" + if candidate_slug not in context.used_container_slugs: + return LibraryContainerLocator( + context.target_library_key, container_type.value, candidate_slug + ) + # It would be extremely unlikely for us to run out of attempts + raise RuntimeError( + f"Unable to find unique slug after {_MAX_UNIQUE_SLUG_ATTEMPTS} attempts for base: {base_slug}" ) @@ -1224,97 +1023,43 @@ def _get_distinct_target_usage_key( title: str, ) -> LibraryUsageLocatorV2: """ - Find a unique key for block_id by appending a unique identifier if necessary. - - Args: - context: The migration context - source_key: The original usage key from the source - component_type: The component type string - olx: The OLX content of the component - - Returns: - A unique LibraryUsageLocatorV2 for the target - - Raises: - ValueError: If source_key is invalid + Figure out the appropriate target component for this block. """ - # Check if we already processed this block and we are not forking. If we are forking, we will - # want a new target key. - if context.is_already_migrated(source_key) and not context.should_fork_strategy: - log.debug(f"Block {source_key} already exists, reusing first existing target") - existing_target = context.get_existing_target(source_key) - block_id = existing_target.component.local_key - - # mypy thinks LibraryUsageLocatorV2 is abstract. It's not. - return LibraryUsageLocatorV2( # type: ignore[abstract] - context.target_library_key, - source_key.block_type, - block_id - ) - + # If we're not forking, then check if this block was part of our past migration. + # (If we are forking, we will always want a new target key). + if not context.should_fork_strategy: + if previous_block_migration := context.previous_block_migrations.get(source_key): + if isinstance(previous_block_migration, data.ModulestoreBlockMigrationSuccess): + if isinstance(previous_block_migration.target_key, LibraryUsageLocatorV2): + return previous_block_migration.target_key # Generate new unique block ID base_slug = ( source_key.block_id if context.preserve_url_slugs else (slugify(title) or source_key.block_id) ) - unique_slug = _find_unique_slug(context, base_slug, component_type) - - # mypy thinks LibraryUsageLocatorV2 is abstract. It's not. - return LibraryUsageLocatorV2( # type: ignore[abstract] - context.target_library_key, - source_key.block_type, - unique_slug + # Use base base slug if available + base_key = LibraryUsageLocatorV2( # type: ignore[abstract] + context.target_library_key, component_type.name, base_slug ) - - -def _find_unique_slug( - context: _MigrationContext, - base_slug: str, - component_type: ComponentType | None = None, - max_attempts: int = 1000 -) -> str: - """ - Find a unique slug by appending incrementing numbers if necessary. - Using batch querying to avoid multiple database roundtrips. - - Args: - component_type: The component type to check against - base_slug: The base slug to make unique - max_attempts: Maximum number of attempts to prevent infinite loops - - Returns: - A unique slug string - - Raises: - RuntimeError: If unable to find unique slug within max_attempts - """ - if not component_type: - base_key = base_slug - else: - base_key = f"{component_type}:{base_slug}" - - existing_publishable_entity_keys = context.get_existing_target_entity_keys(base_key) - - # Check if base slug is available - if base_key not in existing_publishable_entity_keys: - return base_slug - + if base_key not in context.used_component_keys: + return base_key # Try numbered variations until we find one that doesn't exist - for i in range(1, max_attempts + 1): + for i in range(1, _MAX_UNIQUE_SLUG_ATTEMPTS + 1): candidate_slug = f"{base_slug}_{i}" - candidate_key = f"{component_type}:{candidate_slug}" if component_type else candidate_slug - - if candidate_key not in existing_publishable_entity_keys: - return candidate_slug - - raise RuntimeError(f"Unable to find unique slug after {max_attempts} attempts for base: {base_slug}") + candidate_key = LibraryUsageLocatorV2( # type: ignore[abstract] + context.target_library_key, component_type.name, candidate_slug + ) + if candidate_key not in context.used_component_keys: + return candidate_key + # It would be extremely unlikely for us to run out of attempts + raise RuntimeError(f"Unable to find unique slug after {_MAX_UNIQUE_SLUG_ATTEMPTS} attempts for base: {base_slug}") def _create_migration_artifacts_incrementally( root_migrated_node: _MigratedNode, - source: ModulestoreSource, - migration: ModulestoreMigration, + source: models.ModulestoreSource, + migration: models.ModulestoreMigration, status: UserTaskStatus, source_pk: int | None = None, ) -> None: @@ -1325,17 +1070,34 @@ def _create_migration_artifacts_incrementally( total_nodes = len(nodes) processed = 0 - for source_usage_key, target_version in root_migrated_node.all_source_to_target_pairs(): - block_source, _ = ModulestoreBlockSource.objects.get_or_create( + # Load a mapping from each modified entity's primary key + # to the primary key of the changelog record that captures its modification. + # This will not include any blocks whose migration failed to create a target entity. + entity_pks_to_change_log_record_pks: dict[int, int] = dict( + migration.change_log.records.values_list("entity_id", "id") + ) if migration.change_log else {} + + for source_usage_key, target_version, unsupported_reason in root_migrated_node.all_source_to_target_pairs(): + block_source, _ = models.ModulestoreBlockSource.objects.get_or_create( overall_source=source, key=source_usage_key ) - - ModulestoreBlockMigration.objects.create( - overall_migration=migration, - source=block_source, - target_id=target_version.entity_id, - ) + # target_entity_pk should be None iff the block migration failed + target_entity_pk: int | None = target_version.entity_id if target_version else None + + change_log_record_pk = entity_pks_to_change_log_record_pks.get(target_entity_pk) if target_entity_pk else None + # Only create a migration artifact for this source block if: + # (a) we have a record of a change occuring, or + # (b) it failed. + # If neither a nor b are true, then this source block was skipped. + if change_log_record_pk or unsupported_reason: + models.ModulestoreBlockMigration.objects.create( + overall_migration=migration, + source=block_source, + target_id=target_entity_pk, + change_log_record_id=change_log_record_pk, + unsupported_reason=unsupported_reason, + ) processed += 1 if processed % 10 == 0 or processed == total_nodes: diff --git a/cms/djangoapps/modulestore_migrator/tests/test_api.py b/cms/djangoapps/modulestore_migrator/tests/test_api.py index d208ff85e375..c9e2fc3b587b 100644 --- a/cms/djangoapps/modulestore_migrator/tests/test_api.py +++ b/cms/djangoapps/modulestore_migrator/tests/test_api.py @@ -3,11 +3,10 @@ """ import pytest -from opaque_keys.edx.locator import LibraryLocatorV2 +from opaque_keys.edx.locator import LibraryLocator, LibraryLocatorV2 from openedx_learning.api import authoring as authoring_api from organizations.tests.factories import OrganizationFactory -from cms.djangoapps.contentstore.tests.test_libraries import LibraryTestCase from cms.djangoapps.modulestore_migrator import api from cms.djangoapps.modulestore_migrator.data import CompositionLevel, RepeatHandlingStrategy from cms.djangoapps.modulestore_migrator.models import ModulestoreMigration @@ -15,32 +14,50 @@ from common.djangoapps.student.tests.factories import UserFactory from openedx.core.djangoapps.content_libraries import api as lib_api +from xmodule.modulestore.tests.utils import ModuleStoreTestCase from xmodule.modulestore.tests.factories import BlockFactory, LibraryFactory @pytest.mark.django_db -class TestModulestoreMigratorAPI(LibraryTestCase): +class TestModulestoreMigratorAPI(ModuleStoreTestCase): """ Test cases for the modulestore migrator API. """ def setUp(self): super().setUp() - - self.organization = OrganizationFactory() - self.lib_key_v2 = LibraryLocatorV2.from_string( - f"lib:{self.organization.short_name}:test-key" - ) - lib_api.create_library( - org=self.organization, - slug=self.lib_key_v2.slug, - title="Test Library", - ) - self.library_v2 = lib_api.ContentLibrary.objects.get(slug=self.lib_key_v2.slug) - self.learning_package = self.library_v2.learning_package - self.blocks = [] - for _ in range(3): - self.blocks.append(self._add_simple_content_block().usage_key) + self.user = UserFactory(password=self.user_password, is_staff=True) + self.organization = OrganizationFactory(name="My Org", short_name="myorg") + self.lib_key_v1 = LibraryLocator.from_string("library-v1:myorg+old") + LibraryFactory.create(org="myorg", library="old", display_name="Old Library", modulestore=self.store) + self.lib_key_v2_1 = LibraryLocatorV2.from_string("lib:myorg:1") + self.lib_key_v2_2 = LibraryLocatorV2.from_string("lib:myorg:2") + lib_api.create_library(org=self.organization, slug="1", title="Test Library 1") + lib_api.create_library(org=self.organization, slug="2", title="Test Library 2") + self.library_v2_1 = lib_api.ContentLibrary.objects.get(slug="1") + self.library_v2_2 = lib_api.ContentLibrary.objects.get(slug="2") + self.learning_package = self.library_v2_1.learning_package + self.learning_package_2 = self.library_v2_2.learning_package + self.source_unit_keys = [ + BlockFactory.create( + display_name=f"Unit {c}", + category="vertical", + location=self.lib_key_v1.make_usage_key("vertical", c), + parent_location=self.lib_key_v1.make_usage_key("library", "library"), + user_id=self.user.id, publish_item=False, + ).usage_key for c in ["X", "Y", "Z"] + ] + self.source_html_keys = [ + BlockFactory.create( + display_name=f"HTML {c}", + category="html", + location=self.lib_key_v1.make_usage_key("html", c), + parent_location=self.lib_key_v1.make_usage_key("vertical", c), + user_id=self.user.id, publish_item=False, + ).usage_key for c in ["X", "Y", "Z"] + ] + # We load this last so that it has an updated list of children. + self.lib_v1 = self.store.get_library(self.lib_key_v1) def test_start_migration_to_library(self): """ @@ -52,10 +69,10 @@ def test_start_migration_to_library(self): api.start_migration_to_library( user=user, source_key=source.key, - target_library_key=self.library_v2.library_key, + target_library_key=self.library_v2_1.library_key, target_collection_slug=None, - composition_level=CompositionLevel.Component.value, - repeat_handling_strategy=RepeatHandlingStrategy.Skip.value, + composition_level=CompositionLevel.Component, + repeat_handling_strategy=RepeatHandlingStrategy.Skip, preserve_url_slugs=True, forward_source_to_target=False, ) @@ -81,10 +98,10 @@ def test_start_bulk_migration_to_library(self): api.start_bulk_migration_to_library( user=user, source_key_list=[source.key, source_2.key], - target_library_key=self.library_v2.library_key, + target_library_key=self.library_v2_1.library_key, target_collection_slug_list=None, - composition_level=CompositionLevel.Component.value, - repeat_handling_strategy=RepeatHandlingStrategy.Skip.value, + composition_level=CompositionLevel.Component, + repeat_handling_strategy=RepeatHandlingStrategy.Skip, preserve_url_slugs=True, forward_source_to_target=False, ) @@ -128,10 +145,10 @@ def test_start_migration_to_library_with_collection(self): api.start_migration_to_library( user=user, source_key=source.key, - target_library_key=self.library_v2.library_key, + target_library_key=self.library_v2_1.library_key, target_collection_slug=collection_key, - composition_level=CompositionLevel.Component.value, - repeat_handling_strategy=RepeatHandlingStrategy.Skip.value, + composition_level=CompositionLevel.Component, + repeat_handling_strategy=RepeatHandlingStrategy.Skip, preserve_url_slugs=True, forward_source_to_target=False, ) @@ -157,9 +174,9 @@ def test_start_migration_to_library_with_strategy_skip(self): api.start_migration_to_library( user=user, source_key=source.key, - target_library_key=self.library_v2.library_key, - composition_level=CompositionLevel.Component.value, - repeat_handling_strategy=RepeatHandlingStrategy.Skip.value, + target_library_key=self.library_v2_1.library_key, + composition_level=CompositionLevel.Component, + repeat_handling_strategy=RepeatHandlingStrategy.Skip, preserve_url_slugs=True, forward_source_to_target=False, ) @@ -167,7 +184,7 @@ def test_start_migration_to_library_with_strategy_skip(self): modulestoremigration = ModulestoreMigration.objects.get() assert modulestoremigration.repeat_handling_strategy == RepeatHandlingStrategy.Skip.value - migrated_components = lib_api.get_library_components(self.library_v2.library_key) + migrated_components = lib_api.get_library_components(self.library_v2_1.library_key) assert len(migrated_components) == 1 # Update the block, changing its name @@ -178,9 +195,9 @@ def test_start_migration_to_library_with_strategy_skip(self): api.start_migration_to_library( user=user, source_key=source.key, - target_library_key=self.library_v2.library_key, - composition_level=CompositionLevel.Component.value, - repeat_handling_strategy=RepeatHandlingStrategy.Skip.value, + target_library_key=self.library_v2_1.library_key, + composition_level=CompositionLevel.Component, + repeat_handling_strategy=RepeatHandlingStrategy.Skip, preserve_url_slugs=True, forward_source_to_target=False, ) @@ -189,11 +206,11 @@ def test_start_migration_to_library_with_strategy_skip(self): assert modulestoremigration is not None assert modulestoremigration.repeat_handling_strategy == RepeatHandlingStrategy.Skip.value - migrated_components_fork = lib_api.get_library_components(self.library_v2.library_key) + migrated_components_fork = lib_api.get_library_components(self.library_v2_1.library_key) assert len(migrated_components_fork) == 1 component = lib_api.LibraryXBlockMetadata.from_component( - self.library_v2.library_key, migrated_components_fork[0] + self.library_v2_1.library_key, migrated_components_fork[0] ) assert component.display_name == "Original Block" @@ -215,9 +232,9 @@ def test_start_migration_to_library_with_strategy_update(self): api.start_migration_to_library( user=user, source_key=source.key, - target_library_key=self.library_v2.library_key, - composition_level=CompositionLevel.Component.value, - repeat_handling_strategy=RepeatHandlingStrategy.Skip.value, + target_library_key=self.library_v2_1.library_key, + composition_level=CompositionLevel.Component, + repeat_handling_strategy=RepeatHandlingStrategy.Skip, preserve_url_slugs=True, forward_source_to_target=False, ) @@ -225,7 +242,7 @@ def test_start_migration_to_library_with_strategy_update(self): modulestoremigration = ModulestoreMigration.objects.get() assert modulestoremigration.repeat_handling_strategy == RepeatHandlingStrategy.Skip.value - migrated_components = lib_api.get_library_components(self.library_v2.library_key) + migrated_components = lib_api.get_library_components(self.library_v2_1.library_key) assert len(migrated_components) == 1 # Update the block, changing its name @@ -236,9 +253,9 @@ def test_start_migration_to_library_with_strategy_update(self): api.start_migration_to_library( user=user, source_key=source.key, - target_library_key=self.library_v2.library_key, - composition_level=CompositionLevel.Component.value, - repeat_handling_strategy=RepeatHandlingStrategy.Update.value, + target_library_key=self.library_v2_1.library_key, + composition_level=CompositionLevel.Component, + repeat_handling_strategy=RepeatHandlingStrategy.Update, preserve_url_slugs=True, forward_source_to_target=False, ) @@ -247,11 +264,11 @@ def test_start_migration_to_library_with_strategy_update(self): assert modulestoremigration is not None assert modulestoremigration.repeat_handling_strategy == RepeatHandlingStrategy.Update.value - migrated_components_fork = lib_api.get_library_components(self.library_v2.library_key) + migrated_components_fork = lib_api.get_library_components(self.library_v2_1.library_key) assert len(migrated_components_fork) == 1 component = lib_api.LibraryXBlockMetadata.from_component( - self.library_v2.library_key, migrated_components_fork[0] + self.library_v2_1.library_key, migrated_components_fork[0] ) assert component.display_name == "Updated Block" @@ -273,9 +290,9 @@ def test_start_migration_to_library_with_strategy_forking(self): api.start_migration_to_library( user=user, source_key=source.key, - target_library_key=self.library_v2.library_key, - composition_level=CompositionLevel.Component.value, - repeat_handling_strategy=RepeatHandlingStrategy.Skip.value, + target_library_key=self.library_v2_1.library_key, + composition_level=CompositionLevel.Component, + repeat_handling_strategy=RepeatHandlingStrategy.Skip, preserve_url_slugs=True, forward_source_to_target=False, ) @@ -283,7 +300,7 @@ def test_start_migration_to_library_with_strategy_forking(self): modulestoremigration = ModulestoreMigration.objects.get() assert modulestoremigration.repeat_handling_strategy == RepeatHandlingStrategy.Skip.value - migrated_components = lib_api.get_library_components(self.library_v2.library_key) + migrated_components = lib_api.get_library_components(self.library_v2_1.library_key) assert len(migrated_components) == 1 # Update the block, changing its name @@ -294,9 +311,9 @@ def test_start_migration_to_library_with_strategy_forking(self): api.start_migration_to_library( user=user, source_key=source.key, - target_library_key=self.library_v2.library_key, - composition_level=CompositionLevel.Component.value, - repeat_handling_strategy=RepeatHandlingStrategy.Fork.value, + target_library_key=self.library_v2_1.library_key, + composition_level=CompositionLevel.Component, + repeat_handling_strategy=RepeatHandlingStrategy.Fork, preserve_url_slugs=True, forward_source_to_target=False, ) @@ -305,16 +322,16 @@ def test_start_migration_to_library_with_strategy_forking(self): assert modulestoremigration is not None assert modulestoremigration.repeat_handling_strategy == RepeatHandlingStrategy.Fork.value - migrated_components_fork = lib_api.get_library_components(self.library_v2.library_key) + migrated_components_fork = lib_api.get_library_components(self.library_v2_1.library_key) assert len(migrated_components_fork) == 2 first_component = lib_api.LibraryXBlockMetadata.from_component( - self.library_v2.library_key, migrated_components_fork[0] + self.library_v2_1.library_key, migrated_components_fork[0] ) assert first_component.display_name == "Original Block" second_component = lib_api.LibraryXBlockMetadata.from_component( - self.library_v2.library_key, migrated_components_fork[1] + self.library_v2_1.library_key, migrated_components_fork[1] ) assert second_component.display_name == "Updated Block" @@ -326,9 +343,9 @@ def test_start_migration_to_library_with_strategy_forking(self): api.start_migration_to_library( user=user, source_key=source.key, - target_library_key=self.library_v2.library_key, - composition_level=CompositionLevel.Component.value, - repeat_handling_strategy=RepeatHandlingStrategy.Fork.value, + target_library_key=self.library_v2_1.library_key, + composition_level=CompositionLevel.Component, + repeat_handling_strategy=RepeatHandlingStrategy.Fork, preserve_url_slugs=True, forward_source_to_target=False, ) @@ -337,74 +354,215 @@ def test_start_migration_to_library_with_strategy_forking(self): assert modulestoremigration is not None assert modulestoremigration.repeat_handling_strategy == RepeatHandlingStrategy.Fork.value - migrated_components_fork = lib_api.get_library_components(self.library_v2.library_key) + migrated_components_fork = lib_api.get_library_components(self.library_v2_1.library_key) assert len(migrated_components_fork) == 3 first_component = lib_api.LibraryXBlockMetadata.from_component( - self.library_v2.library_key, migrated_components_fork[0] + self.library_v2_1.library_key, migrated_components_fork[0] ) assert first_component.display_name == "Original Block" second_component = lib_api.LibraryXBlockMetadata.from_component( - self.library_v2.library_key, migrated_components_fork[1] + self.library_v2_1.library_key, migrated_components_fork[1] ) assert second_component.display_name == "Updated Block" third_component = lib_api.LibraryXBlockMetadata.from_component( - self.library_v2.library_key, migrated_components_fork[2] + self.library_v2_1.library_key, migrated_components_fork[2] ) assert third_component.display_name == "Updated Block Again" - def test_get_migration_info(self): + def test_migration_api_for_various_scenarios(self): """ - Test that the API can retrieve migration info. + Test that get_migrations, get_block_migrations, forward_context, and forward_block + behave as expected throughout a convoluted series of intertwined migrations. + + Also, ensure that each of the aforementioned api functions only performs 1 query each. """ + # pylint: disable=too-many-statements user = UserFactory() - collection_key = "test-collection" + all_source_usage_keys = {*self.source_html_keys, *self.source_unit_keys} + all_source_usage_key_strs = {str(sk) for sk in all_source_usage_keys} + + # In this test, we will be migrating self.lib_v1 a total of 6 times. + # We will migrate it to each collection (A, B, and C) twice. + + # Lib 1 has Collection A and Collection B + # Lib 2 has Collection C authoring_api.create_collection( learning_package_id=self.learning_package.id, - key=collection_key, - title="Test Collection", + key="test-collection-1a", + title="Test Collection A in Lib 1", + created_by=user.id, + ) + authoring_api.create_collection( + learning_package_id=self.learning_package.id, + key="test-collection-1b", + title="Test Collection B in Lib 1", + created_by=user.id, + ) + authoring_api.create_collection( + learning_package_id=self.learning_package_2.id, + key="test-collection-2c", + title="Test Collection C in Lib 2", created_by=user.id, ) + # No migrations have happened. + # Everything should return None / empty. + assert not list(api.get_migrations(self.lib_key_v1)) + assert not api.get_forwarding(source_key=self.lib_key_v1) + assert not api.get_forwarding_for_blocks(all_source_usage_keys) + + # FOUR MIGRATIONS! + # * Migrate to Lib1.CollA + # * Migrate to Lib1.CollB using FORK strategy + # * Migrate to Lib1.CollA using UPDATE strategy + # * Migrate to Lib2.CollC + # Note: None of these are forwarding migrations! api.start_migration_to_library( user=user, - source_key=self.lib_key, - target_library_key=self.library_v2.library_key, - target_collection_slug=collection_key, - composition_level=CompositionLevel.Component.value, - repeat_handling_strategy=RepeatHandlingStrategy.Skip.value, + source_key=self.lib_key_v1, + target_library_key=self.lib_key_v2_1, + target_collection_slug="test-collection-1a", + composition_level=CompositionLevel.Unit, + # repeat_handling_strategy here is arbitrary, as there will be no repeats. + repeat_handling_strategy=RepeatHandlingStrategy.Skip, preserve_url_slugs=True, - forward_source_to_target=True, + forward_source_to_target=False, + ) + api.start_migration_to_library( + user=user, + source_key=self.lib_key_v1, + target_library_key=self.lib_key_v2_1, + target_collection_slug="test-collection-1b", + composition_level=CompositionLevel.Unit, + # this will create a 2nd copy of every block + repeat_handling_strategy=RepeatHandlingStrategy.Fork, + preserve_url_slugs=True, + forward_source_to_target=False, + ) + api.start_migration_to_library( + user=user, + source_key=self.lib_key_v1, + target_library_key=self.lib_key_v2_1, + target_collection_slug="test-collection-1a", + composition_level=CompositionLevel.Unit, + # this will update the 2nd copies, but put them in the same collection as the first copies + repeat_handling_strategy=RepeatHandlingStrategy.Update, + preserve_url_slugs=True, + forward_source_to_target=False, ) + api.start_migration_to_library( + user=user, + source_key=self.lib_key_v1, + target_library_key=self.lib_key_v2_2, + target_collection_slug="test-collection-2c", + composition_level=CompositionLevel.Unit, + # repeat_handling_strategy here is arbitrary, as there will be no repeats. + repeat_handling_strategy=RepeatHandlingStrategy.Skip, + preserve_url_slugs=True, + forward_source_to_target=False, + ) + # get_migrations returns in reverse chronological order with self.assertNumQueries(1): - result = api.get_migration_info([self.lib_key]) - row = result.get(self.lib_key) - assert row is not None - assert row.migrations__target__key == str(self.lib_key_v2) - assert row.migrations__target__title == "Test Library" - assert row.migrations__target_collection__key == collection_key - assert row.migrations__target_collection__title == "Test Collection" - - def test_get_target_block_usage_keys(self): - """ - Test that the API can get the list of target block usage keys for a given library. - """ - user = UserFactory() - + migration_2c_i, migration_1a_ii, migration_1b_i, migration_1a_i = api.get_migrations(self.lib_key_v1) + assert not migration_1a_i.is_failed + assert not migration_1b_i.is_failed + assert not migration_1a_ii.is_failed + assert not migration_2c_i.is_failed + # Confirm that the metadata came back correctly. + assert migration_1a_i.target_key == self.lib_key_v2_1 + assert migration_1a_i.target_title == "Test Library 1" + assert migration_1a_i.target_collection_slug == "test-collection-1a" + assert migration_1a_i.target_collection_title == "Test Collection A in Lib 1" + assert migration_2c_i.target_key == self.lib_key_v2_2 + assert migration_2c_i.target_title == "Test Library 2" + assert migration_2c_i.target_collection_slug == "test-collection-2c" + assert migration_2c_i.target_collection_title == "Test Collection C in Lib 2" + # Call get_migration_blocks on each of the four migrations. Convert the mapping + # from UsageKey->BlockMigrationResult into str->str just so we can assert things more easily. + with self.assertNumQueries(1): + mappings_1a_i = { + str(sk): str(bm.target_key) for sk, bm in api.get_migration_blocks(migration_1a_i.pk).items() + } + mappings_1b_i = { + str(sk): str(bm.target_key) for sk, bm in api.get_migration_blocks(migration_1b_i.pk).items() + } + mappings_1a_ii = { + str(sk): str(bm.target_key) for sk, bm in api.get_migration_blocks(migration_1a_ii.pk).items() + } + mappings_2c_i = { + str(sk): str(bm.target_key) for sk, bm in api.get_migration_blocks(migration_2c_i.pk).items() + } + # Each migration should have migrated every source block. + assert set(mappings_1a_i.keys()) == all_source_usage_key_strs + assert set(mappings_1b_i.keys()) == all_source_usage_key_strs + assert set(mappings_1a_ii.keys()) == all_source_usage_key_strs + assert set(mappings_2c_i.keys()) == all_source_usage_key_strs + # Because the migration to Lib1.CollB used FORK, we expect that there is nothing in + # common between it and the prior migration to Lib1.CollA. + assert not (set(mappings_1a_i.values()) & set(mappings_1b_i.values())) + # Because the second migration to Lib1.CollA used UPDATE, we expect that it + # will have all the same mappings as the prior migration to Lib1.CollB. + # This is a little countertuitive, since the migrations targeted different collections, + # but the rule that the migrator follows is "UPDATE uses the block from the most recent migration". + assert mappings_1b_i == mappings_1a_ii + # Since forward_source_to_target=False, we have had no authoritative migration yet. + assert api.get_forwarding(self.lib_key_v1) is None + assert not api.get_forwarding_for_blocks(all_source_usage_keys) + + # ANOTHER MIGRATION! + # * Migrate to Lib2.CollC using UPDATE strategy + # Note: This *is* a forwarding migration api.start_migration_to_library( user=user, - source_key=self.lib_key, - target_library_key=self.library_v2.library_key, - target_collection_slug=None, - composition_level=CompositionLevel.Component.value, - repeat_handling_strategy=RepeatHandlingStrategy.Skip.value, + source_key=self.lib_key_v1, + target_library_key=self.lib_key_v2_2, + target_collection_slug="test-collection-2c", + composition_level=CompositionLevel.Unit, + repeat_handling_strategy=RepeatHandlingStrategy.Update, preserve_url_slugs=True, forward_source_to_target=True, ) + migration_2c_ii, _2c_i, _1a_ii, _1b_i, migration_1a_i_reloaded = api.get_migrations(self.lib_key_v1) + assert migration_1a_i_reloaded.pk == migration_1a_i.pk + assert not migration_2c_ii.is_failed + # Our source lib should now forward to Lib2. + with self.assertNumQueries(1): + forwarded = api.get_forwarding(self.lib_key_v1) + assert forwarded.target_key == self.lib_key_v2_2 + assert forwarded.target_collection_slug == "test-collection-2c" + assert forwarded.pk == migration_2c_ii.pk + # Our source lib's blocks should now forward to ones in Lib2. with self.assertNumQueries(1): - result = api.get_target_block_usage_keys(self.lib_key) - for key in self.blocks: - assert result.get(key) is not None + forwarded_blocks = api.get_forwarding_for_blocks(all_source_usage_keys) + assert forwarded_blocks[self.source_html_keys[1]].target_key.context_key == self.lib_key_v2_2 + assert forwarded_blocks[self.source_unit_keys[1]].target_key.context_key == self.lib_key_v2_2 + + # FINAL MIGRATION! + # * Migrate to Lib1.CollB using UPDATE strategy + # Note: This *is* a forwarding migration, and should supplant the previous + # migration for forwarding purposes. + api.start_migration_to_library( + user=user, + source_key=self.lib_key_v1, + target_library_key=self.lib_key_v2_1, + target_collection_slug="test-collection-1b", + composition_level=CompositionLevel.Unit, + repeat_handling_strategy=RepeatHandlingStrategy.Update, + preserve_url_slugs=True, + forward_source_to_target=True, + ) + migration_1b_ii, _2c_ii, _2c_i, _1a_ii, _1b_i, _1a_i = api.get_migrations(self.lib_key_v1) + assert not migration_1b_ii.is_failed + # Our source lib should now forward to Lib1. + forwarded = api.get_forwarding(self.lib_key_v1) + assert forwarded.target_key == self.lib_key_v2_1 + assert forwarded.target_collection_slug == "test-collection-1b" + assert forwarded.pk == migration_1b_ii.pk + # Our source lib should now forward to Lib1. + forwarded_blocks = api.get_forwarding_for_blocks(all_source_usage_keys) + assert forwarded_blocks[self.source_html_keys[1]].target_key.context_key == self.lib_key_v2_1 + assert forwarded_blocks[self.source_unit_keys[1]].target_key.context_key == self.lib_key_v2_1 diff --git a/cms/djangoapps/modulestore_migrator/tests/test_rest_api.py b/cms/djangoapps/modulestore_migrator/tests/test_rest_api.py new file mode 100644 index 000000000000..4a7f842902db --- /dev/null +++ b/cms/djangoapps/modulestore_migrator/tests/test_rest_api.py @@ -0,0 +1,822 @@ +""" +Unit tests for the modulestore_migrator REST API v1 views. + +These tests focus on validation, HTTP status codes, and serialization/deserialization. +Business logic is mocked out. +""" +from unittest.mock import MagicMock, patch +from uuid import uuid4 + +from django.contrib.auth import get_user_model +from django.test import TestCase +from opaque_keys.edx.locator import CourseLocator +from organizations.tests.factories import OrganizationFactory +from rest_framework import status +from rest_framework.exceptions import PermissionDenied +from rest_framework.test import APIRequestFactory, force_authenticate +from user_tasks.models import UserTaskStatus + +from cms.djangoapps.modulestore_migrator.models import ( + ModulestoreMigration as ModulestoreMigrationModel, + ModulestoreSource, +) +from cms.djangoapps.modulestore_migrator.rest_api.v1.views import ( + BulkMigrationViewSet, + MigrationViewSet, +) +from openedx.core.djangoapps.content_libraries import api as lib_api + + +User = get_user_model() + + +class TestMigrationViewSetCreate(TestCase): + """ + Test the MigrationViewSet.create() endpoint. + + Focus: validation, return codes, serialization/deserialization. + """ + + def setUp(self): + """Set up test fixtures.""" + self.factory = APIRequestFactory() + self.view = MigrationViewSet.as_view({'post': 'create'}) + + # Create test user + self.user = User.objects.create_user( + username='testuser', + email='testuser@test.com', + password='password' + ) + + @patch('cms.djangoapps.modulestore_migrator.rest_api.v1.views.migrator_api') + @patch('cms.djangoapps.modulestore_migrator.rest_api.v1.views.lib_api') + @patch('cms.djangoapps.modulestore_migrator.rest_api.v1.views.auth') + @patch('cms.djangoapps.modulestore_migrator.rest_api.v1.views.UserTaskStatus') + def test_create_migration_success_with_minimal_data( + self, mock_user_task_status, mock_auth, mock_lib_api, mock_migrator_api + ): + """ + Test successful migration creation with minimal required fields. + + Validates: + - 201 status code is returned + - Response contains expected serialized fields + - Request data is properly deserialized + - Permission checks are performed for both source and target + """ + mock_auth.has_studio_write_access.return_value = True + mock_lib_api.require_permission_for_library_key.return_value = None + + mock_task = MagicMock(autospec=True) + mock_task.id = 'test-task-id' + mock_migrator_api.start_migration_to_library.return_value = mock_task + + mock_task_status = MagicMock(autospec=True) + mock_task_status.uuid = uuid4() + mock_task_status.state = 'Pending' + mock_task_status.state_text = 'Pending' + mock_task_status.completed_steps = 0 + mock_task_status.total_steps = 10 + mock_task_status.attempts = 1 + mock_task_status.created = '2025-01-01T00:00:00Z' + mock_task_status.modified = '2025-01-01T00:00:00Z' + mock_task_status.artifacts = [] + mock_task_status.migrations.all.return_value = [] + + mock_user_task_status.objects.get.return_value = mock_task_status + + request_data = { + 'source': 'course-v1:TestOrg+TestCourse+TestRun', + 'target': 'lib:TestOrg:TestLibrary', + } + request = self.factory.post( + '/api/modulestore_migrator/v1/migrations/', + data=request_data, + format='json' + ) + force_authenticate(request, user=self.user) + + response = self.view(request) + + assert response.status_code == status.HTTP_201_CREATED + + assert 'uuid' in response.data + assert 'state' in response.data + assert 'state_text' in response.data + assert 'completed_steps' in response.data + assert 'total_steps' in response.data + assert 'parameters' in response.data + + mock_auth.has_studio_write_access.assert_called_once() + mock_lib_api.require_permission_for_library_key.assert_called_once() + + mock_migrator_api.start_migration_to_library.assert_called_once() + call_kwargs = mock_migrator_api.start_migration_to_library.call_args[1] + assert call_kwargs['user'] == self.user + assert str(call_kwargs['source_key']) == 'course-v1:TestOrg+TestCourse+TestRun' + assert str(call_kwargs['target_library_key']) == 'lib:TestOrg:TestLibrary' + + def test_create_migration_invalid_source_key(self): + """ + Test that invalid source key returns 400 Bad Request. + + Validates: + - 400 status code is returned + - Error message mentions validation failure + """ + request_data = { + 'source': 'not-a-valid-key', + 'target': 'lib:TestOrg:TestLibrary', + } + request = self.factory.post( + '/api/modulestore_migrator/v1/migrations/', + data=request_data, + format='json' + ) + force_authenticate(request, user=self.user) + + response = self.view(request) + + assert response.status_code == status.HTTP_400_BAD_REQUEST + assert 'source' in response.data + + def test_create_migration_invalid_target_key(self): + """ + Test that invalid target library key returns 400 Bad Request. + + Validates: + - 400 status code is returned + - Error message mentions target validation failure + """ + request_data = { + 'source': 'course-v1:TestOrg+TestCourse+TestRun', + 'target': 'not-a-valid-library-key', + } + request = self.factory.post( + '/api/modulestore_migrator/v1/migrations/', + data=request_data, + format='json' + ) + force_authenticate(request, user=self.user) + + response = self.view(request) + + assert response.status_code == status.HTTP_400_BAD_REQUEST + assert 'target' in response.data + + def test_create_migration_missing_required_fields(self): + """ + Test that missing required fields returns 400 Bad Request. + + Validates: + - 400 status code is returned when source is missing + - 400 status code is returned when target is missing + """ + request_data = { + 'target': 'lib:TestOrg:TestLibrary', + } + request = self.factory.post( + '/api/modulestore_migrator/v1/migrations/', + data=request_data, + format='json' + ) + force_authenticate(request, user=self.user) + response = self.view(request) + + assert response.status_code == status.HTTP_400_BAD_REQUEST + assert 'source' in response.data + + request_data = { + 'source': 'course-v1:TestOrg+TestCourse+TestRun', + } + request = self.factory.post( + '/api/modulestore_migrator/v1/migrations/', + data=request_data, + format='json' + ) + force_authenticate(request, user=self.user) + response = self.view(request) + + assert response.status_code == status.HTTP_400_BAD_REQUEST + assert 'target' in response.data + + def test_create_migration_unauthenticated_user(self): + """ + Test that unauthenticated requests return 401 Unauthorized. + + Validates: + - 401 status code for unauthenticated requests + """ + request_data = { + 'source': 'course-v1:TestOrg+TestCourse+TestRun', + 'target': 'lib:TestOrg:TestLibrary', + } + request = self.factory.post( + '/api/modulestore_migrator/v1/migrations/', + data=request_data, + format='json' + ) + # Note: No force_authenticate call + + response = self.view(request) + + assert response.status_code in [status.HTTP_401_UNAUTHORIZED, status.HTTP_403_FORBIDDEN] + + @patch('cms.djangoapps.modulestore_migrator.rest_api.v1.views.auth') + def test_create_migration_without_source_author_access(self, mock_auth): + """ + Test that users without author access to source cannot create migrations. + + Validates: + - 403 Forbidden status code when user lacks author access to source + """ + mock_auth.has_studio_write_access.return_value = False + + request_data = { + 'source': 'course-v1:TestOrg+TestCourse+TestRun', + 'target': 'lib:TestOrg:TestLibrary', + } + request = self.factory.post( + '/api/modulestore_migrator/v1/migrations/', + data=request_data, + format='json' + ) + force_authenticate(request, user=self.user) + + response = self.view(request) + + assert response.status_code == status.HTTP_403_FORBIDDEN + + @patch('cms.djangoapps.modulestore_migrator.rest_api.v1.views.lib_api') + @patch('cms.djangoapps.modulestore_migrator.rest_api.v1.views.auth') + def test_create_migration_without_target_write_access(self, mock_auth, mock_lib_api): + """ + Test that users without write access to target cannot create migrations. + + Validates: + - 403 Forbidden status code when user lacks write access to target library + """ + mock_auth.has_studio_write_access.return_value = True + mock_lib_api.require_permission_for_library_key.side_effect = PermissionDenied( + "User lacks permission to manage content in this library" + ) + + request_data = { + 'source': 'course-v1:TestOrg+TestCourse+TestRun', + 'target': 'lib:TestOrg:TestLibrary', + } + request = self.factory.post( + '/api/modulestore_migrator/v1/migrations/', + data=request_data, + format='json' + ) + force_authenticate(request, user=self.user) + + response = self.view(request) + + assert response.status_code == status.HTTP_403_FORBIDDEN + + @patch('cms.djangoapps.modulestore_migrator.rest_api.v1.views.migrator_api') + @patch('cms.djangoapps.modulestore_migrator.rest_api.v1.views.lib_api') + @patch('cms.djangoapps.modulestore_migrator.rest_api.v1.views.auth') + @patch('cms.djangoapps.modulestore_migrator.rest_api.v1.views.UserTaskStatus') + def test_create_migration_with_optional_fields( + self, mock_user_task_status, mock_auth, mock_lib_api, mock_migrator_api + ): + """ + Test migration creation with all optional fields provided. + + Validates: + - Optional fields are properly deserialized + - Default values are not used when explicit values provided + """ + mock_auth.has_studio_write_access.return_value = True + mock_lib_api.require_permission_for_library_key.return_value = None + + mock_task = MagicMock(autospec=True) + mock_task.id = 'test-task-id' + mock_migrator_api.start_migration_to_library.return_value = mock_task + + mock_task_status = MagicMock(autospec=True) + mock_task_status.uuid = uuid4() + mock_task_status.state = 'Pending' + mock_task_status.state_text = 'Pending' + mock_task_status.completed_steps = 0 + mock_task_status.total_steps = 10 + mock_task_status.attempts = 1 + mock_task_status.created = '2025-01-01T00:00:00Z' + mock_task_status.modified = '2025-01-01T00:00:00Z' + mock_task_status.artifacts = [] + mock_task_status.migrations.all.return_value = [] + + mock_user_task_status.objects.get.return_value = mock_task_status + + request_data = { + 'source': 'course-v1:TestOrg+TestCourse+TestRun', + 'target': 'lib:TestOrg:TestLibrary', + 'target_collection_slug': 'my-collection', + 'composition_level': 'unit', + 'repeat_handling_strategy': 'update', + 'preserve_url_slugs': False, + 'forward_source_to_target': True, + } + request = self.factory.post( + '/api/modulestore_migrator/v1/migrations/', + data=request_data, + format='json' + ) + force_authenticate(request, user=self.user) + + response = self.view(request) + + assert response.status_code == status.HTTP_201_CREATED + + mock_migrator_api.start_migration_to_library.assert_called_once() + call_kwargs = mock_migrator_api.start_migration_to_library.call_args[1] + assert call_kwargs['target_collection_slug'] == 'my-collection' + # CompositionLevel and RepeatHandlingStrategy are enums + assert call_kwargs['composition_level'].value == 'unit' + assert call_kwargs['repeat_handling_strategy'].value == 'update' + assert call_kwargs['preserve_url_slugs'] is False + assert call_kwargs['forward_source_to_target'] is True + + def test_create_migration_invalid_composition_level(self): + """ + Test that invalid composition_level returns 400 Bad Request. + + Validates: + - 400 status code for invalid enum value + """ + request_data = { + 'source': 'course-v1:TestOrg+TestCourse+TestRun', + 'target': 'lib:TestOrg:TestLibrary', + 'composition_level': 'invalid_level', + } + request = self.factory.post( + '/api/modulestore_migrator/v1/migrations/', + data=request_data, + format='json' + ) + force_authenticate(request, user=self.user) + + response = self.view(request) + + assert response.status_code == status.HTTP_400_BAD_REQUEST + assert 'composition_level' in response.data + + def test_create_migration_invalid_repeat_handling_strategy(self): + """ + Test that invalid repeat_handling_strategy returns 400 Bad Request. + + Validates: + - 400 status code for invalid enum value + """ + request_data = { + 'source': 'course-v1:TestOrg+TestCourse+TestRun', + 'target': 'lib:TestOrg:TestLibrary', + 'repeat_handling_strategy': 'invalid_strategy', + } + request = self.factory.post( + '/api/modulestore_migrator/v1/migrations/', + data=request_data, + format='json' + ) + force_authenticate(request, user=self.user) + + response = self.view(request) + + assert response.status_code == status.HTTP_400_BAD_REQUEST + assert 'repeat_handling_strategy' in response.data + + +class TestMigrationViewSetList(TestCase): + """ + Test the MigrationViewSet.list() endpoint. + + Focus: validation, return codes, serialization/deserialization. + """ + + def setUp(self): + """Set up test fixtures.""" + self.factory = APIRequestFactory() + self.view = MigrationViewSet.as_view({'get': 'list'}) + + self.user = User.objects.create_user( + username='testuser', + email='testuser@test.com', + password='password' + ) + self.other_user = User.objects.create_user( + username='otheruser', + email='otheruser@test.com', + password='password' + ) + + def test_list_migrations_success(self): + """ + Test successful listing of migrations for the authenticated user. + + Validates: + - 200 status code is returned + - Response contains list of migrations + - Only user's own migrations are returned (other users' migrations filtered out) + """ + org = OrganizationFactory(short_name="TestOrg", name="Test Org") + source_key = CourseLocator.from_string('course-v1:TestOrg+TestCourse+TestRun') + source = ModulestoreSource.objects.create(key=str(source_key)) + target = lib_api.create_library(org=org, slug="TestLib", title="Test Target Lib") + user_task_status = UserTaskStatus.objects.create( + user=self.user, + task_id='user-task-id', + task_class='test.Task', + name='User Migration', + total_steps=10, + completed_steps=10, + ) + other_task_status = UserTaskStatus.objects.create( + user=self.other_user, + task_id='other-task-id', + task_class='test.Task', + name='Other Migration', + total_steps=5, + completed_steps=5, + ) + ModulestoreMigrationModel.objects.create( + task_status=user_task_status, + source=source, + target_id=target.learning_package_id, + ) + ModulestoreMigrationModel.objects.create( + task_status=other_task_status, + source=source, + target_id=target.learning_package_id, + ) + + request = self.factory.get('/api/modulestore_migrator/v1/migrations/') + force_authenticate(request, user=self.user) + response = self.view(request) + + assert response.status_code == status.HTTP_200_OK + results = response.data['results'] + assert len(results) == 1 + assert results[0]['uuid'] == str(user_task_status.uuid) + + def test_list_migrations_unauthenticated(self): + """ + Test that unauthenticated requests return 401 Unauthorized. + + Validates: + - 401 status code for unauthenticated requests + """ + request = self.factory.get('/api/modulestore_migrator/v1/migrations/') + + response = self.view(request) + + assert response.status_code == status.HTTP_401_UNAUTHORIZED + + +class TestMigrationViewSetRetrieve(TestCase): + """ + Test the MigrationViewSet.retrieve() endpoint. + + Focus: validation, return codes, serialization/deserialization. + """ + + def setUp(self): + """Set up test fixtures.""" + self.factory = APIRequestFactory() + self.view = MigrationViewSet.as_view({'get': 'retrieve'}) + + self.user = User.objects.create_user( + username='testuser', + email='testuser@test.com', + password='password' + ) + + def test_retrieve_migration_success(self): + """ + Test successful retrieval of a specific migration by UUID. + + Validates: + - 200 status code is returned + - Response contains migration details + """ + org = OrganizationFactory(short_name="TestOrg", name="Test Org") + source_key = CourseLocator.from_string('course-v1:TestOrg+TestCourse+TestRun') + source = ModulestoreSource.objects.create(key=str(source_key)) + target = lib_api.create_library(org=org, slug="TestLib", title="Test Target Lib") + user_task_status = UserTaskStatus.objects.create( + user=self.user, + task_id='user-task-id', + task_class='test.Task', + name='User Migration', + total_steps=10, + completed_steps=10, + ) + ModulestoreMigrationModel.objects.create( + task_status=user_task_status, + source=source, + target_id=target.learning_package_id, + ) + + request = self.factory.get(f'/api/modulestore_migrator/v1/migrations/{user_task_status.uuid}/') + force_authenticate(request, user=self.user) + response = self.view(request, uuid=str(user_task_status.uuid)) + + assert response.status_code == status.HTTP_200_OK + assert response.data['uuid'] == str(user_task_status.uuid) + assert 'parameters' in response.data + + def test_retrieve_migration_other_user(self): + """ + Test that users cannot retrieve migrations created by other users. + + Validates: + - 404 status code when attempting to retrieve another user's migration + - Users are isolated to their own migrations + """ + other_user = User.objects.create_user( + username='otheruser', + email='other@test.com', + password='password' + ) + org = OrganizationFactory(short_name="TestOrg", name="Test Org") + source_key = CourseLocator.from_string('course-v1:TestOrg+TestCourse+TestRun') + source = ModulestoreSource.objects.create(key=str(source_key)) + target = lib_api.create_library(org=org, slug="TestLib", title="Test Target Lib") + other_task_status = UserTaskStatus.objects.create( + user=other_user, + task_id='other-task-id', + task_class='test.Task', + name='Other Migration', + total_steps=10, + completed_steps=10, + ) + ModulestoreMigrationModel.objects.create( + task_status=other_task_status, + source=source, + target_id=target.learning_package_id, + ) + + request = self.factory.get(f'/api/modulestore_migrator/v1/migrations/{other_task_status.uuid}/') + force_authenticate(request, user=self.user) + response = self.view(request, uuid=str(other_task_status.uuid)) + + assert response.status_code == status.HTTP_404_NOT_FOUND + + def test_retrieve_migration_unauthenticated(self): + """ + Test that unauthenticated requests return 401 Unauthorized. + + Validates: + - 401 status code for unauthenticated requests + """ + task_uuid = uuid4() + request = self.factory.get(f'/api/modulestore_migrator/v1/migrations/{task_uuid}/') + + response = self.view(request, uuid=str(task_uuid)) + + assert response.status_code == status.HTTP_401_UNAUTHORIZED + + +class TestMigrationViewSetCancel(TestCase): + """ + Test the MigrationViewSet.cancel() endpoint. + + Focus: validation, return codes, authorization. + """ + + def setUp(self): + """Set up test fixtures.""" + self.factory = APIRequestFactory() + self.view = MigrationViewSet.as_view({'post': 'cancel'}) + + self.staff_user = User.objects.create_user( + username='staffuser', + email='staff@test.com', + password='password', + is_staff=True + ) + self.regular_user = User.objects.create_user( + username='regularuser', + email='regular@test.com', + password='password', + is_staff=False + ) + + def test_cancel_migration_as_staff(self): + """ + Test that staff users can cancel migrations. + + Validates: + - Staff users can successfully cancel migrations + - UserTaskStatus.cancel is called + """ + org = OrganizationFactory(short_name="TestOrg", name="Test Org") + source_key = CourseLocator.from_string('course-v1:TestOrg+TestCourse+TestRun') + source = ModulestoreSource.objects.create(key=str(source_key)) + target = lib_api.create_library(org=org, slug="TestLib", title="Test Target Lib") + user_task_status = UserTaskStatus.objects.create( + user=self.staff_user, + task_id='staff-task-id', + task_class='test.Task', + name='Staff Migration', + total_steps=10, + completed_steps=5, + ) + ModulestoreMigrationModel.objects.create( + task_status=user_task_status, + source=source, + target_id=target.learning_package_id, + ) + + with patch.object(UserTaskStatus, 'cancel') as mock_cancel: + request = self.factory.post( + f'/api/modulestore_migrator/v1/migrations/{user_task_status.uuid}/cancel/' + ) + force_authenticate(request, user=self.staff_user) + response = self.view(request, uuid=str(user_task_status.uuid)) + + assert response.status_code == status.HTTP_200_OK + mock_cancel.assert_called_once() + + def test_cancel_migration_not_found(self): + """ + Test that attempting to cancel a non-existent migration returns 404. + + Validates: + - 404 status code when migration UUID does not exist + """ + nonexistent_uuid = uuid4() + request = self.factory.post( + f'/api/modulestore_migrator/v1/migrations/{nonexistent_uuid}/cancel/' + ) + force_authenticate(request, user=self.staff_user) + + response = self.view(request, uuid=str(nonexistent_uuid)) + + assert response.status_code == status.HTTP_404_NOT_FOUND + + def test_cancel_migration_as_non_staff(self): + """ + Test that non-staff users cannot cancel migrations. + + Validates: + - 403 Forbidden status code for non-staff users + """ + task_uuid = uuid4() + request = self.factory.post( + f'/api/modulestore_migrator/v1/migrations/{task_uuid}/cancel/' + ) + force_authenticate(request, user=self.regular_user) + + response = self.view(request, uuid=str(task_uuid)) + + assert response.status_code == status.HTTP_403_FORBIDDEN + + def test_cancel_migration_unauthenticated(self): + """ + Test that unauthenticated users cannot cancel migrations. + """ + task_uuid = uuid4() + request = self.factory.post( + f'/api/modulestore_migrator/v1/migrations/{task_uuid}/cancel/' + ) + + response = self.view(request, uuid=str(task_uuid)) + + assert response.status_code == status.HTTP_401_UNAUTHORIZED + + +class TestBulkMigrationViewSetCreate(TestCase): + """ + Test the BulkMigrationViewSet.create() endpoint. + + Focus: validation, return codes, serialization/deserialization. + """ + + def setUp(self): + """Set up test fixtures.""" + self.factory = APIRequestFactory() + self.view = BulkMigrationViewSet.as_view({'post': 'create'}) + + self.user = User.objects.create_user( + username='testuser', + email='testuser@test.com', + password='password' + ) + + @patch('cms.djangoapps.modulestore_migrator.rest_api.v1.views.migrator_api') + @patch('cms.djangoapps.modulestore_migrator.rest_api.v1.views.lib_api') + @patch('cms.djangoapps.modulestore_migrator.rest_api.v1.views.auth') + @patch('cms.djangoapps.modulestore_migrator.rest_api.v1.views.UserTaskStatus') + def test_create_bulk_migration_success( + self, mock_user_task_status, mock_auth, mock_lib_api, mock_migrator_api + ): + """ + Test successful bulk migration creation with multiple sources. + + Validates: + - 201 status code is returned + - Response contains expected serialized fields + - Multiple sources are properly deserialized + """ + mock_auth.has_studio_write_access.return_value = True + mock_lib_api.require_permission_for_library_key.return_value = None + + mock_task = MagicMock(autospec=True) + mock_task.id = 'test-task-id' + mock_migrator_api.start_bulk_migration_to_library.return_value = mock_task + + mock_task_status = MagicMock(autospec=True) + mock_task_status.uuid = uuid4() + mock_task_status.state = 'Pending' + mock_task_status.state_text = 'Pending' + mock_task_status.completed_steps = 0 + mock_task_status.total_steps = 10 + mock_task_status.attempts = 1 + mock_task_status.created = '2025-01-01T00:00:00Z' + mock_task_status.modified = '2025-01-01T00:00:00Z' + mock_task_status.artifacts = [] + mock_task_status.migrations.all.return_value = [] + + mock_user_task_status.objects.get.return_value = mock_task_status + + request_data = { + 'sources': [ + 'course-v1:TestOrg+TestCourse1+Run1', + 'course-v1:TestOrg+TestCourse2+Run2' + ], + 'target': 'lib:TestOrg:TestLibrary', + } + request = self.factory.post( + '/api/modulestore_migrator/v1/bulk_migration/', + data=request_data, + format='json' + ) + force_authenticate(request, user=self.user) + + response = self.view(request) + + assert response.status_code == status.HTTP_201_CREATED + assert 'uuid' in response.data + assert 'parameters' in response.data + + mock_migrator_api.start_bulk_migration_to_library.assert_called_once() + call_kwargs = mock_migrator_api.start_bulk_migration_to_library.call_args[1] + assert call_kwargs['source_key_list'] == [ + CourseLocator.from_string('course-v1:TestOrg+TestCourse1+Run1'), + CourseLocator.from_string('course-v1:TestOrg+TestCourse2+Run2'), + ] + assert call_kwargs['target_collection_slug_list'] is None + assert call_kwargs['create_collections'] is False + # CompositionLevel and RepeatHandlingStrategy are enums + assert call_kwargs['composition_level'].value == 'component' + assert call_kwargs['repeat_handling_strategy'].value == 'skip' + assert call_kwargs['preserve_url_slugs'] is False + assert call_kwargs['forward_source_to_target'] is None + + def test_create_bulk_migration_invalid_source_key(self): + """ + Test that invalid source key in list returns 400 Bad Request. + + Validates: + - 400 status code when one or more sources are invalid + """ + request_data = { + 'sources': ['not-a-valid-key', 'course-v1:TestOrg+TestCourse+TestRun'], + 'target': 'lib:TestOrg:TestLibrary', + } + request = self.factory.post( + '/api/modulestore_migrator/v1/bulk_migration/', + data=request_data, + format='json' + ) + force_authenticate(request, user=self.user) + + response = self.view(request) + + assert response.status_code == status.HTTP_400_BAD_REQUEST + assert 'sources' in response.data + + def test_create_bulk_migration_missing_sources(self): + """ + Test that missing sources field returns 400 Bad Request. + + Validates: + - 400 status code when sources is missing + """ + request_data = { + 'target': 'lib:TestOrg:TestLibrary', + } + request = self.factory.post( + '/api/modulestore_migrator/v1/bulk_migration/', + data=request_data, + format='json' + ) + force_authenticate(request, user=self.user) + response = self.view(request) + + assert response.status_code == status.HTTP_400_BAD_REQUEST + assert 'sources' in response.data diff --git a/cms/djangoapps/modulestore_migrator/tests/test_tasks.py b/cms/djangoapps/modulestore_migrator/tests/test_tasks.py index 244d435de50a..02ac7c6f2bbc 100644 --- a/cms/djangoapps/modulestore_migrator/tests/test_tasks.py +++ b/cms/djangoapps/modulestore_migrator/tests/test_tasks.py @@ -3,44 +3,45 @@ """ from unittest.mock import Mock, patch + import ddt from django.utils import timezone from lxml import etree from opaque_keys.edx.keys import CourseKey from opaque_keys.edx.locator import LibraryLocator, LibraryLocatorV2 -from openedx_learning.api.authoring_models import Collection, PublishableEntityVersion from openedx_learning.api import authoring as authoring_api +from openedx_learning.api.authoring_models import Collection, PublishableEntityVersion from organizations.tests.factories import OrganizationFactory from user_tasks.models import UserTaskArtifact from user_tasks.tasks import UserTaskStatus -from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase -from xmodule.modulestore.tests.factories import CourseFactory, LibraryFactory -from common.djangoapps.student.tests.factories import UserFactory from cms.djangoapps.modulestore_migrator.data import CompositionLevel, RepeatHandlingStrategy from cms.djangoapps.modulestore_migrator.models import ( ModulestoreMigration, ModulestoreSource, ) from cms.djangoapps.modulestore_migrator.tasks import ( + MigrationStep, + _BulkMigrationTask, _migrate_component, _migrate_container, _migrate_node, _MigratedNode, _MigrationContext, - _MigrationTask, - _BulkMigrationTask, - migrate_from_modulestore, bulk_migrate_from_modulestore, - MigrationStep, ) +from common.djangoapps.student.tests.factories import UserFactory from openedx.core.djangoapps.content_libraries import api as lib_api +from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase +from xmodule.modulestore.tests.factories import CourseFactory, LibraryFactory, BlockFactory + +from .. import api as migrator_api @ddt.ddt class TestMigrateFromModulestore(ModuleStoreTestCase): """ - Test the migrate_from_modulestore task + Test the bulk_migrate_from_modulestore task """ def setUp(self): @@ -100,6 +101,28 @@ def setUp(self): title="Test Collection 2", ) + def _make_migration_context(self, **kwargs) -> _MigrationContext: + """ + Builds a _MigrationContext object with default values, overridable with kwargs + """ + return _MigrationContext( + **{ + "used_component_keys": set(), + "used_container_slugs": set(), + "previous_block_migrations": {}, + "target_package_id": self.learning_package.id, + "target_library_key": self.library.library_key, + "source_context_key": self.course.id, + "content_by_filename": {}, + "composition_level": CompositionLevel.Unit, + "repeat_handling_strategy": RepeatHandlingStrategy.Skip, + "preserve_url_slugs": True, + "created_at": timezone.now(), + "created_by": self.user.id, + **kwargs, + }, + ) + def _get_task_status_fail_message(self, status): """ Helper method to get the failure message from a UserTaskStatus object. @@ -113,18 +136,7 @@ def test_migrate_node_wiki_tag(self): Test _migrate_node ignores wiki tags """ wiki_node = etree.fromstring("") - context = _MigrationContext( - existing_source_to_target_keys={}, - target_package_id=self.learning_package.id, - target_library_key=self.library.library_key, - source_context_key=self.course.id, - content_by_filename={}, - composition_level=CompositionLevel.Unit, - repeat_handling_strategy=RepeatHandlingStrategy.Skip, - preserve_url_slugs=True, - created_at=timezone.now(), - created_by=self.user.id, - ) + context = self._make_migration_context() result = _migrate_node( context=context, @@ -143,19 +155,7 @@ def test_migrate_node_course_root(self): '' "" ) - context = _MigrationContext( - existing_source_to_target_keys={}, - target_package_id=self.learning_package.id, - target_library_key=self.library.library_key, - source_context_key=self.course.id, - content_by_filename={}, - composition_level=CompositionLevel.Unit, - repeat_handling_strategy=RepeatHandlingStrategy.Skip, - preserve_url_slugs=True, - created_at=timezone.now(), - created_by=self.user.id, - ) - + context = self._make_migration_context() result = _migrate_node( context=context, source_node=course_node, @@ -175,18 +175,7 @@ def test_migrate_node_library_root(self): '' "" ) - context = _MigrationContext( - existing_source_to_target_keys={}, - target_package_id=self.learning_package.id, - target_library_key=self.library.library_key, - source_context_key=self.course.id, - content_by_filename={}, - composition_level=CompositionLevel.Unit, - repeat_handling_strategy=RepeatHandlingStrategy.Skip, - preserve_url_slugs=True, - created_at=timezone.now(), - created_by=self.user.id, - ) + context = self._make_migration_context() result = _migrate_node( context=context, source_node=library_node, @@ -215,19 +204,7 @@ def test_migrate_node_container_composition_level( container_node = etree.fromstring( f'<{tag_name} url_name="test_{tag_name}" display_name="Test {tag_name.title()}" />' ) - context = _MigrationContext( - existing_source_to_target_keys={}, - target_package_id=self.learning_package.id, - target_library_key=self.library.library_key, - source_context_key=self.course.id, - content_by_filename={}, - composition_level=composition_level, - repeat_handling_strategy=RepeatHandlingStrategy.Skip, - preserve_url_slugs=True, - created_at=timezone.now(), - created_by=self.user.id, - ) - + context = self._make_migration_context(composition_level=composition_level) result = _migrate_node( context=context, source_node=container_node, @@ -235,9 +212,10 @@ def test_migrate_node_container_composition_level( if should_migrate: self.assertIsNotNone(result.source_to_target) - source_key, _ = result.source_to_target + source_key, _, reason = result.source_to_target self.assertEqual(source_key.block_type, tag_name) self.assertEqual(source_key.block_id, f"test_{tag_name}") + self.assertIsNone(reason) else: self.assertIsNone(result.source_to_target) @@ -248,25 +226,40 @@ def test_migrate_node_without_url_name(self): node_without_url_name = etree.fromstring( '' ) - context = _MigrationContext( - existing_source_to_target_keys={}, - target_package_id=self.learning_package.id, - target_library_key=self.library.library_key, - source_context_key=self.course.id, - content_by_filename={}, - composition_level=CompositionLevel.Unit, - repeat_handling_strategy=RepeatHandlingStrategy.Skip, - preserve_url_slugs=True, - created_at=timezone.now(), - created_by=self.user.id, + context = self._make_migration_context() + result = _migrate_node( + context=context, + source_node=node_without_url_name, ) + self.assertIsNone(result.source_to_target) + self.assertEqual(len(result.children), 0) + + def test_migrate_node_with_children_components(self): + """ + Test _migrate_node handles nodes with children components + """ + node_without_url_name = etree.fromstring(''' + + + + + ''') + context = self._make_migration_context() result = _migrate_node( context=context, source_node=node_without_url_name, ) - self.assertIsNone(result.source_to_target) + self.assertEqual( + result.source_to_target, + ( + self.course.id.make_usage_key('library_content', 'test_library_content'), + None, + 'The "library_content" XBlock (ID: "test_library_content") has children, ' + 'so it not supported in content libraries. It has 2 children blocks.', + ), + ) self.assertEqual(len(result.children), 0) def test_migrated_node_all_source_to_target_pairs(self): @@ -281,11 +274,11 @@ def test_migrated_node_all_source_to_target_pairs(self): key2 = self.course.id.make_usage_key("problem", "problem2") key3 = self.course.id.make_usage_key("problem", "problem3") - child_node = _MigratedNode(source_to_target=(key3, mock_version3), children=[]) + child_node = _MigratedNode(source_to_target=(key3, mock_version3, None), children=[]) parent_node = _MigratedNode( - source_to_target=(key1, mock_version1), + source_to_target=(key1, mock_version1, None), children=[ - _MigratedNode(source_to_target=(key2, mock_version2), children=[]), + _MigratedNode(source_to_target=(key2, mock_version2, None), children=[]), child_node, ], ) @@ -297,27 +290,6 @@ def test_migrated_node_all_source_to_target_pairs(self): self.assertEqual(pairs[1][0], key2) self.assertEqual(pairs[2][0], key3) - def test_migrate_from_modulestore_invalid_source(self): - """ - Test migrate_from_modulestore with invalid source - """ - task = migrate_from_modulestore.apply_async( - kwargs={ - "user_id": self.user.id, - "source_pk": 999999, # Non-existent source - "target_library_key": str(self.lib_key), - "target_collection_pk": self.collection.id, - "repeat_handling_strategy": RepeatHandlingStrategy.Skip.value, - "preserve_url_slugs": True, - "composition_level": CompositionLevel.Unit.value, - "forward_source_to_target": False, - } - ) - - status = UserTaskStatus.objects.get(task_id=task.id) - self.assertEqual(status.state, UserTaskStatus.FAILED) - self.assertEqual(self._get_task_status_fail_message(status), "ModulestoreSource matching query does not exist.") - def test_bulk_migrate_invalid_sources(self): """ Test bulk_migrate_from_modulestore with invalid source @@ -339,31 +311,6 @@ def test_bulk_migrate_invalid_sources(self): self.assertEqual(status.state, UserTaskStatus.FAILED) self.assertEqual(self._get_task_status_fail_message(status), "ModulestoreSource matching query does not exist.") - def test_migrate_from_modulestore_invalid_collection(self): - """ - Test migrate_from_modulestore with invalid collection - """ - source = ModulestoreSource.objects.create( - key=self.course.id, - ) - - task = migrate_from_modulestore.apply_async( - kwargs={ - "user_id": self.user.id, - "source_pk": source.id, - "target_library_key": str(self.lib_key), - "target_collection_pk": 999999, # Non-existent collection - "repeat_handling_strategy": RepeatHandlingStrategy.Skip.value, - "preserve_url_slugs": True, - "composition_level": CompositionLevel.Unit.value, - "forward_source_to_target": False, - } - ) - - status = UserTaskStatus.objects.get(task_id=task.id) - self.assertEqual(status.state, UserTaskStatus.FAILED) - self.assertEqual(self._get_task_status_fail_message(status), "Collection matching query does not exist.") - def test_bulk_migrate_invalid_collection(self): """ Test bulk_migrate_from_modulestore with invalid collection @@ -389,14 +336,6 @@ def test_bulk_migrate_invalid_collection(self): self.assertEqual(status.state, UserTaskStatus.FAILED) self.assertEqual(self._get_task_status_fail_message(status), "Collection matching query does not exist.") - def test_migration_task_calculate_total_steps(self): - """ - Test _MigrationTask.calculate_total_steps returns correct count - """ - total_steps = _MigrationTask.calculate_total_steps({}) - expected_steps = len(list(MigrationStep)) - 1 - self.assertEqual(total_steps, expected_steps) - def test_bulk_migration_task_calculate_total_steps(self): """ Test _BulkMigrationTask.calculate_total_steps returns correct count @@ -413,26 +352,15 @@ def test_migrate_component_success(self): """ source_key = self.course.id.make_usage_key("problem", "test_problem") olx = '' - context = _MigrationContext( - existing_source_to_target_keys={}, - target_package_id=self.learning_package.id, - target_library_key=self.library.library_key, - source_context_key=self.course.id, - content_by_filename={}, - composition_level=CompositionLevel.Unit, - repeat_handling_strategy=RepeatHandlingStrategy.Skip, - preserve_url_slugs=True, - created_at=timezone.now(), - created_by=self.user.id, - ) - - result = _migrate_component( + context = self._make_migration_context() + result, reason = _migrate_component( context=context, source_key=source_key, olx=olx, title="test_problem" ) + self.assertIsNone(reason) self.assertIsNotNone(result) self.assertIsInstance(result, PublishableEntityVersion) @@ -443,6 +371,32 @@ def test_migrate_component_success(self): # The component is published self.assertFalse(result.componentversion.component.versioning.has_unpublished_changes) + def test_migrate_component_failure(self): + """ + Test _migrate_component fails to import component with children + """ + source_key = self.course.id.make_usage_key("library_content", "test_library_content") + olx = ''' + + + + + ''' + context = self._make_migration_context() + result, reason = _migrate_component( + context=context, + source_key=source_key, + olx=olx, + title="test_library content" + ) + + self.assertIsNone(result) + self.assertEqual( + reason, + 'The "library_content" XBlock (ID: "test_library_content") has children,' + ' so it not supported in content libraries.', + ) + def test_migrate_component_with_static_content(self): """ Test _migrate_component with static file content @@ -458,19 +412,8 @@ def test_migrate_component_with_static_content(self): created=timezone.now(), ) content_by_filename = {"test_image.png": test_content.id} - context = _MigrationContext( - existing_source_to_target_keys={}, - target_package_id=self.learning_package.id, - target_library_key=self.library.library_key, - source_context_key=self.course.id, - content_by_filename=content_by_filename, - composition_level=CompositionLevel.Unit, - repeat_handling_strategy=RepeatHandlingStrategy.Skip, - preserve_url_slugs=True, - created_at=timezone.now(), - created_by=self.user.id, - ) - result = _migrate_component( + context = self._make_migration_context(content_by_filename=content_by_filename) + result, reason = _migrate_component( context=context, source_key=source_key, olx=olx, @@ -478,6 +421,7 @@ def test_migrate_component_with_static_content(self): ) self.assertIsNotNone(result) + self.assertIsNone(reason) component_content = result.componentversion.componentversioncontent_set.filter( key="static/test_image.png" @@ -485,159 +429,204 @@ def test_migrate_component_with_static_content(self): self.assertIsNotNone(component_content) self.assertEqual(component_content.content_id, test_content.id) - def test_migrate_component_replace_existing_false(self): + def test_migrate_skip_repeats(self): """ - Test _migrate_component with replace_existing=False returns existing component + Test that, when requested, the migration will Skip blocks that have previously been migrated + + Tests with both a container and a component """ - source_key = self.course.id.make_usage_key("problem", "existing_problem") - olx = '' - context = _MigrationContext( - existing_source_to_target_keys={}, - target_package_id=self.learning_package.id, - target_library_key=self.library.library_key, - source_context_key=self.course.id, - content_by_filename={}, - composition_level=CompositionLevel.Unit, - repeat_handling_strategy=RepeatHandlingStrategy.Skip, - preserve_url_slugs=True, - created_at=timezone.now(), - created_by=self.user.id, - ) + source = ModulestoreSource.objects.create(key=self.course.id) - first_result = _migrate_component( - context=context, - source_key=source_key, - olx=olx, - title="test_problem" + # Create a legacy lib with 2 blocks and migrate it + source_html = BlockFactory.create( + category="html", + display_name="Test HTML for Skip", + parent_location=self.course.usage_key, + user_id=self.user.id, + publish_item=False + ) + source_unit = BlockFactory.create( + category="vertical", + display_name="Test Unit for Skip", + parent_location=self.course.usage_key, + user_id=self.user.id, + publish_item=False + ) + bulk_migrate_from_modulestore.apply_async( + kwargs={ + "user_id": self.user.id, + "sources_pks": [source.id], + "target_library_key": str(self.lib_key), + "target_collection_pks": [], + "repeat_handling_strategy": RepeatHandlingStrategy.Skip.value, # arbitrary + "preserve_url_slugs": True, + "composition_level": CompositionLevel.Unit.value, + "forward_source_to_target": False, + } ) - context.existing_source_to_target_keys[source_key] = [first_result.entity] - - second_result = _migrate_component( - context=context, - source_key=source_key, - olx='', - title="updated_problem" + # Update both blocks, and add a new one. Then migrate again. + source_html.display_name = "Test HTML for Skip - Source Updated" + source_html.save() + self.store.update_item(source_html, self.user.id) + source_unit.display_name = "Test Unit for Skip - Source Updated" + source_unit.save() + self.store.update_item(source_unit, self.user.id) + source_html_new = BlockFactory.create( + category="html", + display_name="Test HTML New", + parent_location=self.course.usage_key, + user_id=self.user.id, + publish_item=False + ) + bulk_migrate_from_modulestore.apply_async( + kwargs={ + "user_id": self.user.id, + "sources_pks": [source.id], + "target_library_key": str(self.lib_key), + "target_collection_pks": [], + "repeat_handling_strategy": RepeatHandlingStrategy.Skip.value, # <-- important + "preserve_url_slugs": True, + "composition_level": CompositionLevel.Unit.value, + "forward_source_to_target": False, + } ) - self.assertEqual(first_result.entity_id, second_result.entity_id) - self.assertEqual(first_result.version_num, second_result.version_num) + # The first migration's info includes the initial two blocks. + migration_1, migration_0 = list(migrator_api.get_migrations(source_key=source.key)) + mappings_0 = migrator_api.get_migration_blocks(migration_0.pk) + assert set(mappings_0) == {source_html.usage_key, source_unit.usage_key} + assert mappings_0[source_html.usage_key].target_title == "Test HTML for Skip" + assert mappings_0[source_unit.usage_key].target_title == "Test Unit for Skip" + + # The next migration's info includes the newly-added block, + # but not the edited blocks, because we chose Skip. + mappings_1 = migrator_api.get_migration_blocks(migration_1.pk) + assert set(mappings_1) == {source_html_new.usage_key} + assert mappings_1[source_html_new.usage_key].target_title == "Test HTML New" def test_migrate_component_same_title(self): """ - Test _migrate_component for two components with the same title + Test a migration with two components of the same title, when updating. - Using preserve_url_slugs=False to create a new component with - a different URL slug based on the component's Title. + We expect that both blocks will be migrated to target components with usage keys + based on the shared title, but disambiguated by a _1 suffix. """ - source_key_1 = self.course.id.make_usage_key("problem", "existing_problem_1") - source_key_2 = self.course.id.make_usage_key("problem", "existing_problem_2") - olx = '' - context = _MigrationContext( - existing_source_to_target_keys={}, - target_package_id=self.learning_package.id, - target_library_key=self.library.library_key, - source_context_key=self.course.id, - content_by_filename={}, - composition_level=CompositionLevel.Unit, - repeat_handling_strategy=RepeatHandlingStrategy.Skip, - preserve_url_slugs=False, - created_at=timezone.now(), - created_by=self.user.id, - ) - - first_result = _migrate_component( - context=context, - source_key=source_key_1, - olx=olx, - title="test_problem" - ) - - context.existing_source_to_target_keys[source_key_1] = [first_result.entity] - - second_result = _migrate_component( - context=context, - source_key=source_key_2, - olx=olx, - title="test_problem" + source = ModulestoreSource.objects.create(key=self.course.id) + source_key_1 = self.course.id.make_usage_key("html", "existing_html_1") + source_key_2 = self.course.id.make_usage_key("html", "existing_html_2") + BlockFactory.create( + category="html", + display_name="Test HTML Same Title", + location=source_key_1, + parent_location=self.course.usage_key, + user_id=self.user.id, + publish_item=False + ) + BlockFactory.create( + category="html", + display_name="Test HTML Same Title", + location=source_key_2, + parent_location=self.course.usage_key, + user_id=self.user.id, + publish_item=False + ) + bulk_migrate_from_modulestore.apply_async( + kwargs={ + "user_id": self.user.id, + "sources_pks": [source.id], + "target_library_key": str(self.lib_key), + "target_collection_pks": [], + "repeat_handling_strategy": RepeatHandlingStrategy.Skip.value, + "preserve_url_slugs": False, + "composition_level": CompositionLevel.Unit.value, + "forward_source_to_target": False, + } ) + migrations = list(migrator_api.get_migrations(source_key=source.key)) + assert len(migrations) == 1 + mappings = migrator_api.get_migration_blocks(migrations[0].pk) + assert (html_migration_1 := mappings.get(source_key_1)) + assert (block_migration_2 := mappings.get(source_key_2)) + assert html_migration_1.target_title == "Test HTML Same Title" + assert block_migration_2.target_title == "Test HTML Same Title" + assert str(html_migration_1.target_key) == "lb:testorg:test-key:html:test-html-same-title" + assert str(block_migration_2.target_key) == "lb:testorg:test-key:html:test-html-same-title_1" - self.assertNotEqual(first_result.entity_id, second_result.entity_id) - self.assertNotEqual(first_result.entity.key, second_result.entity.key) - - def test_migrate_component_replace_existing_true(self): - """ - Test _migrate_component with replace_existing=True creates new version + def test_migrate_update_repeats(self): """ - source_key = self.course.id.make_usage_key("problem", "replaceable_problem") - original_olx = '' - context = _MigrationContext( - existing_source_to_target_keys={}, - target_package_id=self.learning_package.id, - target_library_key=self.library.library_key, - source_context_key=self.course.id, - content_by_filename={}, - composition_level=CompositionLevel.Unit, - repeat_handling_strategy=RepeatHandlingStrategy.Update, - preserve_url_slugs=True, - created_at=timezone.now(), - created_by=self.user.id, - ) + Test that, when requested, the migration will update blocks that have previously been migrated - first_result = _migrate_component( - context=context, - source_key=source_key, - olx=original_olx, - title="original" + Tests with both a container and a component + """ + source = ModulestoreSource.objects.create(key=self.course.id) + source_html = BlockFactory.create( + category="html", + display_name="Test HTML for Update", + parent_location=self.course.usage_key, + user_id=self.user.id, + publish_item=False + ) + source_unit = BlockFactory.create( + category="vertical", + display_name="Test Unit for Update", + parent_location=self.course.usage_key, + user_id=self.user.id, + publish_item=False + ) + bulk_migrate_from_modulestore.apply_async( + kwargs={ + "user_id": self.user.id, + "sources_pks": [source.id], + "target_library_key": str(self.lib_key), + "target_collection_pks": [], + # (the value of repeat_handling_strategy here doesn't matter for this test) + "repeat_handling_strategy": RepeatHandlingStrategy.Skip.value, + "preserve_url_slugs": True, + "composition_level": CompositionLevel.Unit.value, + "forward_source_to_target": False, + } ) - - context.existing_source_to_target_keys[source_key] = [first_result.entity] - - updated_olx = '' - second_result = _migrate_component( - context=context, - source_key=source_key, - olx=updated_olx, - title="updated" + source_html.display_name = "Test HTML for Update - Source Updated" + source_html.save() + self.store.update_item(source_html, self.user.id) + source_unit.display_name = "Test Unit for Update - Source Updated" + source_unit.save() + self.store.update_item(source_unit, self.user.id) + bulk_migrate_from_modulestore.apply_async( + kwargs={ + "user_id": self.user.id, + "sources_pks": [source.id], + "target_library_key": str(self.lib_key), + "target_collection_pks": [], + "repeat_handling_strategy": RepeatHandlingStrategy.Update.value, + "preserve_url_slugs": True, + "composition_level": CompositionLevel.Unit.value, + "forward_source_to_target": False, + } ) - - self.assertEqual(first_result.entity_id, second_result.entity_id) - self.assertNotEqual(first_result.version_num, second_result.version_num) - - def test_migrate_component_different_block_types(self): - """ - Test _migrate_component with different block types - """ - block_types = ["problem", "html", "video", "discussion"] - - for block_type in block_types: - source_key = self.course.id.make_usage_key(block_type, f"test_{block_type}") - olx = f'<{block_type} display_name="Test {block_type.title()}">' - context = _MigrationContext( - existing_source_to_target_keys={}, - target_package_id=self.learning_package.id, - target_library_key=self.library.library_key, - source_context_key=self.course.id, - content_by_filename={}, - composition_level=CompositionLevel.Unit, - repeat_handling_strategy=RepeatHandlingStrategy.Skip, - preserve_url_slugs=True, - created_at=timezone.now(), - created_by=self.user.id, - ) - - result = _migrate_component( - context=context, - source_key=source_key, - olx=olx, - title="test" - ) - - self.assertIsNotNone(result, f"Failed to migrate {block_type}") - - self.assertEqual( - block_type, result.componentversion.component.component_type.name - ) + migration_1, migration_0 = list(migrator_api.get_migrations(source_key=source.key)) + mappings_0 = migrator_api.get_migration_blocks(migration_0.pk) + mappings_1 = migrator_api.get_migration_blocks(migration_1.pk) + assert (html_migration_0 := mappings_0.get(source_html.usage_key)) + assert (unit_migration_0 := mappings_0.get(source_unit.usage_key)) + assert (html_migration_1 := mappings_1.get(source_html.usage_key)) + assert (unit_migration_1 := mappings_1.get(source_unit.usage_key)) + + # The targets of both migrations are the same + assert str(html_migration_0.target_key) == "lb:testorg:test-key:html:Test_HTML_for_Update" + assert str(html_migration_1.target_key) == "lb:testorg:test-key:html:Test_HTML_for_Update" + assert html_migration_0.target_entity_pk == html_migration_1.target_entity_pk + assert str(unit_migration_0.target_key) == "lct:testorg:test-key:unit:Test_Unit_for_Update" + assert unit_migration_0.target_entity_pk == unit_migration_1.target_entity_pk + + # And because we specified Update, the targets were updated on the 2nd migration + assert html_migration_0.target_title == "Test HTML for Update" + assert unit_migration_0.target_title == "Test Unit for Update" + assert html_migration_1.target_title == "Test HTML for Update - Source Updated" + assert unit_migration_1.target_title == "Test Unit for Update - Source Updated" + assert html_migration_0.target_version_num == html_migration_1.target_version_num - 1 + assert unit_migration_0.target_version_num == unit_migration_1.target_version_num - 1 def test_migrate_component_content_filename_not_in_olx(self): """ @@ -666,20 +655,11 @@ def test_migrate_component_content_filename_not_in_olx(self): "referenced.png": referenced_content.id, "unreferenced.png": unreferenced_content.id, } - context = _MigrationContext( - existing_source_to_target_keys={}, - target_package_id=self.learning_package.id, - target_library_key=self.library.library_key, - source_context_key=self.course.id, + context = self._make_migration_context( content_by_filename=content_by_filename, - composition_level=CompositionLevel.Unit, repeat_handling_strategy=RepeatHandlingStrategy.Skip, - preserve_url_slugs=True, - created_at=timezone.now(), - created_by=self.user.id, ) - - result = _migrate_component( + result, reason = _migrate_component( context=context, source_key=source_key, olx=olx, @@ -687,6 +667,7 @@ def test_migrate_component_content_filename_not_in_olx(self): ) self.assertIsNotNone(result) + self.assertIsNone(reason) referenced_content_exists = ( result.componentversion.componentversioncontent_set.filter( @@ -709,20 +690,8 @@ def test_migrate_component_library_source_key(self): library_key = LibraryLocator(org="TestOrg", library="TestLibrary") source_key = library_key.make_usage_key("problem", "library_problem") olx = '' - context = _MigrationContext( - existing_source_to_target_keys={}, - target_package_id=self.learning_package.id, - target_library_key=self.library.library_key, - source_context_key=self.course.id, - content_by_filename={}, - composition_level=CompositionLevel.Unit, - repeat_handling_strategy=RepeatHandlingStrategy.Skip, - preserve_url_slugs=True, - created_at=timezone.now(), - created_by=self.user.id, - ) - - result = _migrate_component( + context = self._make_migration_context() + result, reason = _migrate_component( context=context, source_key=source_key, olx=olx, @@ -730,61 +699,12 @@ def test_migrate_component_library_source_key(self): ) self.assertIsNotNone(result) + self.assertIsNone(reason) self.assertEqual( "problem", result.componentversion.component.component_type.name ) - def test_migrate_component_duplicate_content_integrity_error(self): - """ - Test _migrate_component handles IntegrityError when content already exists - """ - source_key = self.course.id.make_usage_key( - "problem", "test_problem_duplicate_content" - ) - olx = '

    See image: duplicate.png

    ' - - media_type = authoring_api.get_or_create_media_type("image/png") - test_content = authoring_api.get_or_create_file_content( - self.learning_package.id, - media_type.id, - data=b"test_image_data", - created=timezone.now(), - ) - content_by_filename = {"duplicate.png": test_content.id} - context = _MigrationContext( - existing_source_to_target_keys={}, - target_package_id=self.learning_package.id, - target_library_key=self.library.library_key, - source_context_key=self.course.id, - content_by_filename=content_by_filename, - composition_level=CompositionLevel.Unit, - repeat_handling_strategy=RepeatHandlingStrategy.Update, - preserve_url_slugs=True, - created_at=timezone.now(), - created_by=self.user.id, - ) - - first_result = _migrate_component( - context=context, - source_key=source_key, - olx=olx, - title="test_problem" - ) - - context.existing_source_to_target_keys[source_key] = [first_result.entity] - - second_result = _migrate_component( - context=context, - source_key=source_key, - olx=olx, - title="test_problem" - ) - - self.assertIsNotNone(first_result) - self.assertIsNotNone(second_result) - self.assertEqual(first_result.entity_id, second_result.entity_id) - def test_migrate_container_creates_new_container(self): """ Test _migrate_container creates a new container when none exists @@ -827,20 +747,9 @@ def test_migrate_container_creates_new_container(self): child_version_1.publishable_entity_version, child_version_2.publishable_entity_version, ] - context = _MigrationContext( - existing_source_to_target_keys={}, - target_package_id=self.learning_package.id, - target_library_key=self.library.library_key, - source_context_key=self.course.id, - content_by_filename={}, - composition_level=CompositionLevel.Unit, - repeat_handling_strategy=RepeatHandlingStrategy.Skip, - preserve_url_slugs=True, - created_at=timezone.now(), - created_by=self.user.id, - ) + context = self._make_migration_context(repeat_handling_strategy=RepeatHandlingStrategy.Skip) - result = _migrate_container( + result, reason = _migrate_container( context=context, source_key=source_key, container_type=lib_api.ContainerType.Unit, @@ -848,6 +757,7 @@ def test_migrate_container_creates_new_container(self): children=children, ) + self.assertIsNone(reason) self.assertIsInstance(result, PublishableEntityVersion) container_version = result.containerversion @@ -869,18 +779,7 @@ def test_migrate_container_different_container_types(self): (lib_api.ContainerType.Subsection, "sequential"), (lib_api.ContainerType.Section, "chapter"), ] - context = _MigrationContext( - existing_source_to_target_keys={}, - target_package_id=self.learning_package.id, - target_library_key=self.library.library_key, - source_context_key=self.course.id, - content_by_filename={}, - composition_level=CompositionLevel.Unit, - repeat_handling_strategy=RepeatHandlingStrategy.Skip, - preserve_url_slugs=True, - created_at=timezone.now(), - created_by=self.user.id, - ) + context = self._make_migration_context(repeat_handling_strategy=RepeatHandlingStrategy.Skip) for container_type, block_type in container_types: with self.subTest(container_type=container_type, block_type=block_type): @@ -888,7 +787,7 @@ def test_migrate_container_different_container_types(self): block_type, f"test_{block_type}" ) - result = _migrate_container( + result, reason = _migrate_container( context=context, source_key=source_key, container_type=container_type, @@ -896,6 +795,7 @@ def test_migrate_container_different_container_types(self): children=[], ) + self.assertIsNone(reason) self.assertIsNotNone(result) container_version = result.containerversion @@ -903,175 +803,62 @@ def test_migrate_container_different_container_types(self): # The container is published self.assertFalse(authoring_api.contains_unpublished_changes(container_version.container.pk)) - def test_migrate_container_replace_existing_false(self): - """ - Test _migrate_container returns existing container when replace_existing=False - """ - source_key = self.course.id.make_usage_key("vertical", "existing_vertical") - context = _MigrationContext( - existing_source_to_target_keys={}, - target_package_id=self.learning_package.id, - target_library_key=self.library.library_key, - source_context_key=self.course.id, - content_by_filename={}, - composition_level=CompositionLevel.Unit, - repeat_handling_strategy=RepeatHandlingStrategy.Skip, - preserve_url_slugs=True, - created_at=timezone.now(), - created_by=self.user.id, - ) - - first_result = _migrate_container( - context=context, - source_key=source_key, - container_type=lib_api.ContainerType.Unit, - title="Original Title", - children=[], - ) - - context.existing_source_to_target_keys[source_key] = [first_result.entity] - - second_result = _migrate_container( - context=context, - source_key=source_key, - container_type=lib_api.ContainerType.Unit, - title="Updated Title", - children=[], - ) - - self.assertEqual(first_result.entity_id, second_result.entity_id) - self.assertEqual(first_result.version_num, second_result.version_num) - - container_version = second_result.containerversion - self.assertEqual(container_version.title, "Original Title") - def test_migrate_container_same_title(self): """ - Test _migrate_container for two containers with the same title - - Using preserve_url_slugs=False to create a new Unit with - a different URL slug based on the container's Title. - """ - source_key_1 = self.course.id.make_usage_key("vertical", "human_readable_vertical_1") - source_key_2 = self.course.id.make_usage_key("vertical", "human_readable_vertical_2") - context = _MigrationContext( - existing_source_to_target_keys={}, - target_package_id=self.learning_package.id, - target_library_key=self.library.library_key, - source_context_key=self.course.id, - content_by_filename={}, - composition_level=CompositionLevel.Unit, - repeat_handling_strategy=RepeatHandlingStrategy.Skip, - preserve_url_slugs=False, - created_at=timezone.now(), - created_by=self.user.id, - ) - - first_result = _migrate_container( - context=context, - source_key=source_key_1, - container_type=lib_api.ContainerType.Unit, - title="Original Human Readable Title", - children=[], - ) - - context.existing_source_to_target_keys[source_key_1] = [first_result.entity] - - second_result = _migrate_container( - context=context, - source_key=source_key_2, - container_type=lib_api.ContainerType.Unit, - title="Original Human Readable Title", - children=[], - ) - - self.assertNotEqual(first_result.entity_id, second_result.entity_id) - self.assertNotEqual(first_result.entity.key, second_result.entity.key) - # Make sure the current logic from tasts::_find_unique_slug is used - self.assertEqual(second_result.entity.key, first_result.entity.key + "_1") - - container_version = second_result.containerversion - self.assertEqual(container_version.title, "Original Human Readable Title") - - def test_migrate_container_replace_existing_true(self): - """ - Test _migrate_container creates new version when replace_existing=True - """ - source_key = self.course.id.make_usage_key("vertical", "replaceable_vertical") + Test a migration with two containers of the same title and preserve_url_slugs=False - child_component = authoring_api.create_component( - self.learning_package.id, - component_type=authoring_api.get_or_create_component_type( - "xblock.v1", "problem" - ), - local_key="child_problem", - created=timezone.now(), - created_by=self.user.id, - ) - child_version = authoring_api.create_next_component_version( - child_component.pk, - content_to_replace={}, - created=timezone.now(), - created_by=self.user.id, - ) - context = _MigrationContext( - existing_source_to_target_keys={}, - target_package_id=self.learning_package.id, - target_library_key=self.library.library_key, - source_context_key=self.course.id, - content_by_filename={}, - composition_level=CompositionLevel.Unit, - repeat_handling_strategy=RepeatHandlingStrategy.Update, - preserve_url_slugs=True, - created_at=timezone.now(), - created_by=self.user.id, - ) - - first_result = _migrate_container( - context=context, - source_key=source_key, - container_type=lib_api.ContainerType.Unit, - title="Original Title", - children=[], - ) - - context.existing_source_to_target_keys[source_key] = [first_result.entity] - - second_result = _migrate_container( - context=context, - source_key=source_key, - container_type=lib_api.ContainerType.Unit, - title="Updated Title", - children=[child_version.publishable_entity_version], - ) - - self.assertEqual(first_result.entity_id, second_result.entity_id) - self.assertNotEqual(first_result.version_num, second_result.version_num) - - container_version = second_result.containerversion - self.assertEqual(container_version.title, "Updated Title") - self.assertEqual(container_version.entity_list.entitylistrow_set.count(), 1) - - def test_migrate_container_with_library_source_key(self): + We expect that both units will be migrated to target units with container keys + based on the shared title, but disambiguated by a _1 suffix. """ - Test _migrate_container with library source key - """ - library_key = LibraryLocator(org="TestOrg", library="TestLibrary") - source_key = library_key.make_usage_key("vertical", "library_vertical") - context = _MigrationContext( - existing_source_to_target_keys={}, - target_package_id=self.learning_package.id, - target_library_key=self.library.library_key, - source_context_key=self.course.id, - content_by_filename={}, - composition_level=CompositionLevel.Unit, - repeat_handling_strategy=RepeatHandlingStrategy.Skip, - preserve_url_slugs=True, - created_at=timezone.now(), - created_by=self.user.id, + source = ModulestoreSource.objects.create(key=self.course.id) + source_key_1 = self.course.id.make_usage_key("vertical", "existing_unit_1") + source_key_2 = self.course.id.make_usage_key("vertical", "existing_unit_2") + BlockFactory.create( + category="vertical", + display_name="Test Unit Same Title", + location=source_key_1, + parent_location=self.course.usage_key, + user_id=self.user.id, + publish_item=False + ) + BlockFactory.create( + category="html", + display_name="Test Unit Same Title", + location=source_key_2, + parent_location=self.course.usage_key, + user_id=self.user.id, + publish_item=False + ) + bulk_migrate_from_modulestore.apply_async( + kwargs={ + "user_id": self.user.id, + "sources_pks": [source.id], + "target_library_key": str(self.lib_key), + "target_collection_pks": [], + "repeat_handling_strategy": RepeatHandlingStrategy.Skip.value, + "preserve_url_slugs": False, + "composition_level": CompositionLevel.Unit.value, + "forward_source_to_target": False, + } ) + (migration,) = list(migrator_api.get_migrations(source_key=source.key)) + mappings = migrator_api.get_migration_blocks(migration.pk) + assert (html_migration_1 := mappings.get(source_key_1)) + assert (block_migration_2 := mappings.get(source_key_2)) + assert html_migration_1.target_title == "Test Unit Same Title" + assert block_migration_2.target_title == "Test Unit Same Title" + assert str(html_migration_1.target_key) == "lct:testorg:test-key:unit:test-unit-same-title" + assert str(block_migration_2.target_key) == "lct:testorg:test-key:unit:test-unit-same-title_1" + + def test_migrate_container_with_library_source_key(self): + """ + Test _migrate_container with library source key + """ + library_key = LibraryLocator(org="TestOrg", library="TestLibrary") + source_key = library_key.make_usage_key("vertical", "library_vertical") + context = self._make_migration_context(repeat_handling_strategy=RepeatHandlingStrategy.Skip) - result = _migrate_container( + result, _ = _migrate_container( context=context, source_key=source_key, container_type=lib_api.ContainerType.Unit, @@ -1089,20 +876,8 @@ def test_migrate_container_empty_children_list(self): Test _migrate_container handles empty children list """ source_key = self.course.id.make_usage_key("vertical", "empty_vertical") - context = _MigrationContext( - existing_source_to_target_keys={}, - target_package_id=self.learning_package.id, - target_library_key=self.library.library_key, - source_context_key=self.course.id, - content_by_filename={}, - composition_level=CompositionLevel.Unit, - repeat_handling_strategy=RepeatHandlingStrategy.Skip, - preserve_url_slugs=True, - created_at=timezone.now(), - created_by=self.user.id, - ) - - result = _migrate_container( + context = self._make_migration_context(repeat_handling_strategy=RepeatHandlingStrategy.Skip) + result, reason = _migrate_container( context=context, source_key=source_key, container_type=lib_api.ContainerType.Unit, @@ -1110,6 +885,7 @@ def test_migrate_container_empty_children_list(self): children=[], ) + self.assertIsNone(reason) self.assertIsNotNone(result) container_version = result.containerversion @@ -1120,18 +896,7 @@ def test_migrate_container_preserves_child_order(self): Test _migrate_container preserves the order of children """ source_key = self.course.id.make_usage_key("vertical", "ordered_vertical") - context = _MigrationContext( - existing_source_to_target_keys={}, - target_package_id=self.learning_package.id, - target_library_key=self.library.library_key, - source_context_key=self.course.id, - content_by_filename={}, - composition_level=CompositionLevel.Unit, - repeat_handling_strategy=RepeatHandlingStrategy.Skip, - preserve_url_slugs=True, - created_at=timezone.now(), - created_by=self.user.id, - ) + context = self._make_migration_context(repeat_handling_strategy=RepeatHandlingStrategy.Skip) children = [] for i in range(3): child_component = authoring_api.create_component( @@ -1151,7 +916,7 @@ def test_migrate_container_preserves_child_order(self): ) children.append(child_version.publishable_entity_version) - result = _migrate_container( + result, _ = _migrate_container( context=context, source_key=source_key, container_type=lib_api.ContainerType.Unit, @@ -1227,20 +992,8 @@ def test_migrate_container_with_mixed_child_types(self): html_version.publishable_entity_version, video_version.publishable_entity_version, ] - context = _MigrationContext( - existing_source_to_target_keys={}, - target_package_id=self.learning_package.id, - target_library_key=self.library.library_key, - source_context_key=self.course.id, - content_by_filename={}, - composition_level=CompositionLevel.Unit, - repeat_handling_strategy=RepeatHandlingStrategy.Skip, - preserve_url_slugs=True, - created_at=timezone.now(), - created_by=self.user.id, - ) - - result = _migrate_container( + context = self._make_migration_context(repeat_handling_strategy=RepeatHandlingStrategy.Skip) + result, _ = _migrate_container( context=context, source_key=source_key, container_type=lib_api.ContainerType.Unit, @@ -1261,76 +1014,6 @@ def test_migrate_container_with_mixed_child_types(self): expected_entity_ids = {child.entity_id for child in children} self.assertEqual(child_entity_ids, expected_entity_ids) - def test_migrate_container_generates_correct_target_key(self): - """ - Test _migrate_container generates correct target key from source key - """ - course_source_key = self.course.id.make_usage_key("vertical", "test_vertical") - context = _MigrationContext( - existing_source_to_target_keys={}, - target_package_id=self.learning_package.id, - target_library_key=self.library.library_key, - source_context_key=self.course.id, - content_by_filename={}, - composition_level=CompositionLevel.Unit, - repeat_handling_strategy=RepeatHandlingStrategy.Skip, - preserve_url_slugs=True, - created_at=timezone.now(), - created_by=self.user.id, - ) - - course_result = _migrate_container( - context=context, - source_key=course_source_key, - container_type=lib_api.ContainerType.Unit, - title="Course Vertical", - children=[], - ) - context.add_migration(course_source_key, course_result.entity) - - library_key = LibraryLocator(org="TestOrg", library="TestLibrary") - library_source_key = library_key.make_usage_key("vertical", "test_vertical") - - library_result = _migrate_container( - context=context, - source_key=library_source_key, - container_type=lib_api.ContainerType.Unit, - title="Library Vertical", - children=[], - ) - - self.assertIsNotNone(course_result) - self.assertIsNotNone(library_result) - self.assertNotEqual(course_result.entity_id, library_result.entity_id) - - def test_migrate_from_modulestore_success_course(self): - """ - Test successful migration from course to library - """ - source = ModulestoreSource.objects.create(key=self.course.id) - - task = migrate_from_modulestore.apply_async( - kwargs={ - "user_id": self.user.id, - "source_pk": source.id, - "target_library_key": str(self.lib_key), - "target_collection_pk": self.collection.id, - "repeat_handling_strategy": RepeatHandlingStrategy.Skip.value, - "preserve_url_slugs": True, - "composition_level": CompositionLevel.Unit.value, - "forward_source_to_target": False, - } - ) - - status = UserTaskStatus.objects.get(task_id=task.id) - self.assertEqual(status.state, UserTaskStatus.SUCCEEDED) - - migration = ModulestoreMigration.objects.get( - source=source, target=self.learning_package - ) - self.assertEqual(migration.composition_level, CompositionLevel.Unit.value) - self.assertEqual(migration.repeat_handling_strategy, RepeatHandlingStrategy.Skip.value) - def test_bulk_migrate_success_courses(self): """ Test successful bulk migration from courses to library @@ -1372,12 +1055,12 @@ def test_migrate_from_modulestore_success_legacy_library(self): """ source = ModulestoreSource.objects.create(key=self.legacy_library.location.library_key) - task = migrate_from_modulestore.apply_async( + task = bulk_migrate_from_modulestore.apply_async( kwargs={ "user_id": self.user.id, - "source_pk": source.id, + "sources_pks": [source.id], "target_library_key": str(self.lib_key), - "target_collection_pk": self.collection.id, + "target_collection_pks": [self.collection.id], "repeat_handling_strategy": RepeatHandlingStrategy.Skip.value, "preserve_url_slugs": True, "composition_level": CompositionLevel.Unit.value, @@ -1523,7 +1206,6 @@ def test_bulk_migrate_use_previous_collection_on_skip_and_update(self, repeat_ha migrations = ModulestoreMigration.objects.filter( source=source, target=self.learning_package ) - for migration in migrations: self.assertEqual(migration.composition_level, CompositionLevel.Unit.value) self.assertEqual(migration.repeat_handling_strategy, repeat_handling_strategy.value) @@ -1663,50 +1345,19 @@ def test_bulk_migrate_create_a_new_collection_on_fork(self): self.assertEqual(migrations[1].target_collection.title, f"{self.legacy_library.display_name}_1") self.assertNotEqual(migrations[1].target_collection.id, previous_collection.id) - def test_migrate_from_modulestore_library_validation_failure(self): - """ - Test migration from legacy library fails when modulestore content doesn't exist - """ - library_key = LibraryLocator(org="TestOrg", library="TestLibrary") - - source = ModulestoreSource.objects.create(key=library_key) - - task = migrate_from_modulestore.apply_async( - kwargs={ - "user_id": self.user.id, - "source_pk": source.id, - "target_library_key": str(self.lib_key), - "target_collection_pk": None, - "repeat_handling_strategy": RepeatHandlingStrategy.Update.value, - "preserve_url_slugs": True, - "composition_level": CompositionLevel.Section.value, - "forward_source_to_target": True, - } - ) - - status = UserTaskStatus.objects.get(task_id=task.id) - - # Should fail at loading step since we don't have real modulestore content - self.assertEqual(status.state, UserTaskStatus.FAILED) - self.assertEqual( - self._get_task_status_fail_message(status), - "Failed to load source item 'lib-block-v1:TestOrg+TestLibrary+type@library+block@library' " - "from ModuleStore: library-v1:TestOrg+TestLibrary+branch@library" - ) - - def test_migrate_from_modulestore_invalid_source_key_type(self): + def test_bulk_migrate_invalid_source_key_type(self): """ - Test migration with invalid source key type + Test bulk migration with invalid source key type """ invalid_key = LibraryLocatorV2.from_string("lib:testorg:invalid") source = ModulestoreSource.objects.create(key=invalid_key) - task = migrate_from_modulestore.apply_async( + task = bulk_migrate_from_modulestore.apply_async( kwargs={ "user_id": self.user.id, - "source_pk": source.id, + "sources_pks": [source.id], "target_library_key": str(self.lib_key), - "target_collection_pk": self.collection.id, + "target_collection_pks": [self.collection.id], "repeat_handling_strategy": RepeatHandlingStrategy.Skip.value, "preserve_url_slugs": True, "composition_level": CompositionLevel.Unit.value, @@ -1721,48 +1372,52 @@ def test_migrate_from_modulestore_invalid_source_key_type(self): f"Not a valid source context key: {invalid_key}. Source key must reference a course or a legacy library." ) - def test_bulk_migrate_invalid_source_key_type(self): + def test_migrate_component_with_fake_block_type(self): """ - Test bulk migration with invalid source key type + Test _migrate_component with with_fake_block_type """ - invalid_key = LibraryLocatorV2.from_string("lib:testorg:invalid") - source = ModulestoreSource.objects.create(key=invalid_key) - - task = bulk_migrate_from_modulestore.apply_async( - kwargs={ - "user_id": self.user.id, - "sources_pks": [source.id], - "target_library_key": str(self.lib_key), - "target_collection_pks": [self.collection.id], - "repeat_handling_strategy": RepeatHandlingStrategy.Skip.value, - "preserve_url_slugs": True, - "composition_level": CompositionLevel.Unit.value, - "forward_source_to_target": False, - } + source_key = self.course.id.make_usage_key("fake_block", "test_fake_block") + olx = '' + context = _MigrationContext( + used_component_keys=set(), + used_container_slugs=set(), + previous_block_migrations={}, + target_package_id=self.learning_package.id, + target_library_key=self.library.library_key, + source_context_key=self.course.id, + content_by_filename={}, + composition_level=CompositionLevel.Unit, + repeat_handling_strategy=RepeatHandlingStrategy.Skip, + preserve_url_slugs=True, + created_at=timezone.now(), + created_by=self.user.id, ) - status = UserTaskStatus.objects.get(task_id=task.id) - self.assertEqual(status.state, UserTaskStatus.FAILED) - self.assertEqual( - self._get_task_status_fail_message(status), - f"Not a valid source context key: {invalid_key}. Source key must reference a course or a legacy library." + result, reason = _migrate_component( + context=context, + source_key=source_key, + olx=olx, + title="test" ) - def test_migrate_from_modulestore_nonexistent_modulestore_item(self): + self.assertIsNone(result) + self.assertEqual(reason, "Invalid block type: fake_block") + + def test_bulk_migrate_nonexistent_modulestore_item(self): """ - Test migration when modulestore item doesn't exist + Test bulk migration when modulestore item doesn't exist """ nonexistent_course_key = CourseKey.from_string( "course-v1:NonExistent+Course+Run" ) source = ModulestoreSource.objects.create(key=nonexistent_course_key) - task = migrate_from_modulestore.apply_async( + task = bulk_migrate_from_modulestore.apply_async( kwargs={ "user_id": self.user.id, - "source_pk": source.id, + "sources_pks": [source.id], "target_library_key": str(self.lib_key), - "target_collection_pk": self.collection.id, + "target_collection_pks": [self.collection.id], "repeat_handling_strategy": RepeatHandlingStrategy.Skip.value, "preserve_url_slugs": True, "composition_level": CompositionLevel.Unit.value, @@ -1778,46 +1433,47 @@ def test_migrate_from_modulestore_nonexistent_modulestore_item(self): "from ModuleStore: course-v1:NonExistent+Course+Run+branch@draft-branch" ) - def test_bulk_migrate_nonexistent_modulestore_item(self): + def test_bulk_migrate_nonexistent_library(self): """ - Test bulk migration when modulestore item doesn't exist + Test migration from legacy library fails when modulestore content doesn't exist """ - nonexistent_course_key = CourseKey.from_string( - "course-v1:NonExistent+Course+Run" - ) - source = ModulestoreSource.objects.create(key=nonexistent_course_key) + library_key = LibraryLocator(org="TestOrg", library="TestLibrary") + + source = ModulestoreSource.objects.create(key=library_key) task = bulk_migrate_from_modulestore.apply_async( kwargs={ "user_id": self.user.id, "sources_pks": [source.id], "target_library_key": str(self.lib_key), - "target_collection_pks": [self.collection.id], - "repeat_handling_strategy": RepeatHandlingStrategy.Skip.value, + "target_collection_pks": [None], + "repeat_handling_strategy": RepeatHandlingStrategy.Update.value, "preserve_url_slugs": True, - "composition_level": CompositionLevel.Unit.value, - "forward_source_to_target": False, + "composition_level": CompositionLevel.Section.value, + "forward_source_to_target": True, } ) status = UserTaskStatus.objects.get(task_id=task.id) + + # Should fail at loading step since we don't have real modulestore content self.assertEqual(status.state, UserTaskStatus.FAILED) self.assertEqual( self._get_task_status_fail_message(status), - "Failed to load source item 'block-v1:NonExistent+Course+Run+type@course+block@course' " - "from ModuleStore: course-v1:NonExistent+Course+Run+branch@draft-branch" + "Failed to load source item 'lib-block-v1:TestOrg+TestLibrary+type@library+block@library' " + "from ModuleStore: library-v1:TestOrg+TestLibrary+branch@library" ) - def test_migrate_from_modulestore_task_status_progression(self): + def test_bulk_migrate_from_modulestore_task_status_progression(self): """Test that task status progresses through expected steps""" source = ModulestoreSource.objects.create(key=self.course.id) - task = migrate_from_modulestore.apply_async( + task = bulk_migrate_from_modulestore.apply_async( kwargs={ "user_id": self.user.id, - "source_pk": source.id, + "sources_pks": [source.id], "target_library_key": str(self.lib_key), - "target_collection_pk": self.collection.id, + "target_collection_pks": [self.collection.id], "repeat_handling_strategy": RepeatHandlingStrategy.Skip.value, "preserve_url_slugs": True, "composition_level": CompositionLevel.Unit.value, @@ -1835,48 +1491,6 @@ def test_migrate_from_modulestore_task_status_progression(self): ) self.assertEqual(migration.task_status, status) - def test_migrate_from_modulestore_multiple_users_no_interference(self): - """ - Test that migrations by different users don't interfere with each other - """ - source = ModulestoreSource.objects.create(key=self.course.id) - other_user = UserFactory() - - task1 = migrate_from_modulestore.apply_async( - kwargs={ - "user_id": self.user.id, - "source_pk": source.id, - "target_library_key": str(self.lib_key), - "target_collection_pk": self.collection.id, - "repeat_handling_strategy": RepeatHandlingStrategy.Skip.value, - "preserve_url_slugs": True, - "composition_level": CompositionLevel.Unit.value, - "forward_source_to_target": False, - } - ) - - task2 = migrate_from_modulestore.apply_async( - kwargs={ - "user_id": other_user.id, - "source_pk": source.id, - "target_library_key": str(self.lib_key), - "target_collection_pk": self.collection.id, - "repeat_handling_strategy": RepeatHandlingStrategy.Skip.value, - "preserve_url_slugs": True, - "composition_level": CompositionLevel.Unit.value, - "forward_source_to_target": False, - } - ) - - status1 = UserTaskStatus.objects.get(task_id=task1.id) - status2 = UserTaskStatus.objects.get(task_id=task2.id) - - self.assertEqual(status1.user, self.user) - self.assertEqual(status2.user, other_user) - - # The first task should not be cancelled since it's from a different user - self.assertNotEqual(status1.state, UserTaskStatus.CANCELED) - def test_bulk_migrate_multiple_users_no_interference(self): """ Test that migrations by different users don't interfere with each other @@ -1919,35 +1533,6 @@ def test_bulk_migrate_multiple_users_no_interference(self): # The first task should not be cancelled since it's from a different user self.assertNotEqual(status1.state, UserTaskStatus.CANCELED) - @patch("cms.djangoapps.modulestore_migrator.tasks._import_assets") - def test_migrate_fails_on_import(self, mock_import_assets): - """ - Test failed migration from legacy library to V2 library - """ - mock_import_assets.side_effect = Exception("Simulated import error") - source = ModulestoreSource.objects.create(key=self.legacy_library.location.library_key) - - task = migrate_from_modulestore.apply_async( - kwargs={ - "user_id": self.user.id, - "source_pk": source.id, - "target_library_key": str(self.lib_key), - "target_collection_pk": self.collection.id, - "repeat_handling_strategy": RepeatHandlingStrategy.Skip.value, - "preserve_url_slugs": True, - "composition_level": CompositionLevel.Unit.value, - "forward_source_to_target": False, - } - ) - - status = UserTaskStatus.objects.get(task_id=task.id) - self.assertEqual(status.state, UserTaskStatus.FAILED) - - migration = ModulestoreMigration.objects.get( - source=source, target=self.learning_package - ) - self.assertTrue(migration.is_failed) - @patch("cms.djangoapps.modulestore_migrator.tasks._import_assets") def test_bulk_migrate_fails_on_import(self, mock_import_assets): """ diff --git a/openedx/core/djangoapps/content_libraries/api/blocks.py b/openedx/core/djangoapps/content_libraries/api/blocks.py index be5b296147fd..1328db6e6ca1 100644 --- a/openedx/core/djangoapps/content_libraries/api/blocks.py +++ b/openedx/core/djangoapps/content_libraries/api/blocks.py @@ -309,7 +309,7 @@ def validate_can_add_block_to_library( block_class = XBlock.load_class(block_type) # Will raise an exception if invalid if block_class.has_children: raise IncompatibleTypesError( - 'The "{block_type}" XBlock (ID: "{block_id}") has children, so it not supported in content libraries', + f'The "{block_type}" XBlock (ID: "{block_id}") has children, so it not supported in content libraries.', ) # Make sure the new ID is not taken already: usage_key = LibraryUsageLocatorV2( # type: ignore[abstract] diff --git a/setup.cfg b/setup.cfg index e4419bd149a9..06aea046d413 100644 --- a/setup.cfg +++ b/setup.cfg @@ -166,6 +166,7 @@ ignore_imports = name = Do not depend on non-public API of isolated apps. type = isolated_apps isolated_apps = + cms.djangoapps.modulestore_migrator openedx.core.djangoapps.agreements openedx.core.djangoapps.bookmarks openedx.core.djangoapps.content_libraries diff --git a/xmodule/library_content_block.py b/xmodule/library_content_block.py index f3d563a7fd30..5853b3923f46 100644 --- a/xmodule/library_content_block.py +++ b/xmodule/library_content_block.py @@ -12,11 +12,12 @@ import json import logging +import typing as t from gettext import gettext, ngettext import nh3 from django.core.exceptions import ObjectDoesNotExist, PermissionDenied -from opaque_keys.edx.locator import LibraryLocator +from opaque_keys.edx.locator import LibraryLocator, LibraryLocatorV2 from web_fragments.fragment import Fragment from webob import Response from xblock.core import XBlock @@ -29,6 +30,10 @@ from xmodule.validation import StudioValidation, StudioValidationMessage from xmodule.x_module import XModuleToXBlockMixin +if t.TYPE_CHECKING: + from xmodule.library_tools import LegacyLibraryToolsService + + _ = lambda text: text logger = logging.getLogger(__name__) @@ -118,20 +123,20 @@ def source_library_key(self): return LibraryLocator.from_string(self.source_library_id) @property - def is_source_lib_migrated_to_v2(self): + def is_source_lib_migrated_to_v2(self) -> bool: """ Determines whether the source library has been migrated to v2. """ - from cms.djangoapps.modulestore_migrator.api import is_successfully_migrated + from cms.djangoapps.modulestore_migrator.api import is_forwarded return ( self.source_library_id and self.source_library_version - and is_successfully_migrated(self.source_library_key, source_version=self.source_library_version) + and is_forwarded(self.source_library_key) ) @property - def is_ready_to_migrated_to_v2(self): + def is_ready_to_migrated_to_v2(self) -> bool: """ Returns whether the block can be migrated to v2. """ @@ -198,7 +203,7 @@ def non_editable_metadata_fields(self): ]) return non_editable_fields - def get_tools(self, to_read_library_content: bool = False) -> 'LegacyLibraryToolsService': + def get_tools(self, to_read_library_content: bool = False) -> LegacyLibraryToolsService: """ Grab the library tools service and confirm that it'll work for us. Else, raise LibraryToolsUnavailable. """ @@ -315,16 +320,20 @@ def _v2_update_children_upstream_version(self): Update the upstream and upstream version fields of all children to point to library v2 version of the legacy library blocks. This essentially converts this legacy block to new ItemBankBlock. """ - from cms.djangoapps.modulestore_migrator.api import get_target_block_usage_keys - blocks = get_target_block_usage_keys(self.source_library_key) + from cms.djangoapps.modulestore_migrator import api as migrator_api store = modulestore() with store.bulk_operations(self.course_id): - for child in self.get_children(): - source_key, _ = self.runtime.modulestore.get_block_original_usage(child.usage_key) - child.upstream = str(blocks.get(source_key, "")) - # Since after migration, the component in library is in draft state, we want to make sure that sync icon - # appears when it is published - child.upstream_version = 0 + children = self.get_children() + child_migrations = migrator_api.get_forwarding_for_blocks([child.usage_key for child in children]) + for child in children: + old_upstream_key, _ = self.runtime.modulestore.get_block_original_usage(child.usage_key) + upstream_migration = child_migrations.get(old_upstream_key) + if upstream_migration and isinstance(upstream_migration.target_key, LibraryLocatorV2): + child.upstream = str(upstream_migration.target_key) + if upstream_migration.target_version_num: + child.upstream_version = upstream_migration.target_version_num + else: + child.upstream = "" # Use `modulestore()` instead of `self.runtime.modulestore` to make sure that the XBLOCK_UPDATED signal # is triggered store.update_item(child, None) diff --git a/xmodule/tests/test_library_content.py b/xmodule/tests/test_library_content.py index 01d27fad98d0..0f7807d61bc8 100644 --- a/xmodule/tests/test_library_content.py +++ b/xmodule/tests/test_library_content.py @@ -762,8 +762,8 @@ def setUp(self): source_key=self.library.location.library_key, target_library_key=self.library_v2.library_key, target_collection_slug=None, - composition_level=CompositionLevel.Component.value, - repeat_handling_strategy=RepeatHandlingStrategy.Skip.value, + composition_level=CompositionLevel.Component, + repeat_handling_strategy=RepeatHandlingStrategy.Skip, preserve_url_slugs=True, forward_source_to_target=True, ) @@ -791,48 +791,3 @@ def test_author_view(self): assert '
  • html 2
    • ' in rendered.content assert '
  • html 3
    • ' in rendered.content assert '
  • html 4
    • ' in rendered.content - - def test_xml_export_import_cycle(self): - """ - Test the export-import cycle. - """ - # Render block to migrate it first - self.lc_block.render(AUTHOR_VIEW, {}) - # Set the virtual FS to export the olx to. - export_fs = MemoryFS() - self.lc_block.runtime.export_fs = export_fs # pylint: disable=protected-access - - # Export the olx. - node = etree.Element("unknown_root") - self.lc_block.add_xml_to_node(node) - - # Read back the olx. - file_path = f'{self.lc_block.scope_ids.usage_id.block_type}/{self.lc_block.scope_ids.usage_id.block_id}.xml' - with export_fs.open(file_path) as f: - exported_olx = f.read() - - expected_olx_export = ( - f'\n' - f' \n' - f' \n' - f' \n' - f' \n' - '\n' - ) - # And compare. - assert exported_olx == expected_olx_export - - # Now import it. - runtime = DummyModuleStoreRuntime(load_error_blocks=True, course_id=self.lc_block.location.course_key) - runtime.resources_fs = export_fs - olx_element = etree.fromstring(exported_olx) - imported_lc_block = LegacyLibraryContentBlock.parse_xml(olx_element, runtime, None) - - self._verify_xblock_properties(imported_lc_block) - # Verify migration info in the child - assert imported_lc_block.is_migrated_to_v2 - for child in imported_lc_block.get_children(): - assert child.xml_attributes.get('upstream') is not None - assert str(child.xml_attributes.get('upstream_version')) == '0' From 4319ccaa6df7d37550c2399fe6261a606858ed42 Mon Sep 17 00:00:00 2001 From: Muhammad Arslan Date: Mon, 5 Jan 2026 17:33:34 +0500 Subject: [PATCH 159/351] fix: remove text from base query_params (#37836) --- lms/djangoapps/discussion/rest_api/api.py | 1 - 1 file changed, 1 deletion(-) diff --git a/lms/djangoapps/discussion/rest_api/api.py b/lms/djangoapps/discussion/rest_api/api.py index 1c0b05735208..443f9527acda 100644 --- a/lms/djangoapps/discussion/rest_api/api.py +++ b/lms/djangoapps/discussion/rest_api/api.py @@ -1018,7 +1018,6 @@ def get_thread_list( "group_id": group_id, "page": page, "per_page": page_size, - "text": text_search, "sort_key": cc_map.get(order_by), "author_id": author_id, "flagged": flagged, From 17703dc4eeb96fdc4e0b20ee2d3d6bebbb8a6b10 Mon Sep 17 00:00:00 2001 From: David Ormsbee Date: Mon, 5 Jan 2026 20:33:45 -0500 Subject: [PATCH 160/351] fix: correct upstream field for migrated libraries (#37838) This is to fix an issue in the following common migration situation: 1. An existing course references content in a legacy content library. 2. The legacy content library is migrated to the new library system. 3. The user clicks on "Update reference" from the Randomized Content Block in the course. This action is supposed to update the children of the LibraryContentBlock (usually ProblemBlocks) so that the "upstream" attribute is set to point at the UsageKeys of the content in the new libraries they were migrated to. What was happening instead was that the upstream entries for these child blocks were left blank, breaking the upstream/sync connection and making it so that the courses did not receive any updates from the migrated libraries. There were two issues: 1. get_forwarding_for_blocks() was being called with the child UsageKeys in the course, when it should have been called with the v1 library usage keys instead (since those are the things being forwarded). 2. We were checking that the target_key was a v2 Library key, but really the upstream target_key is supposed to be a LibraryUsageLocatorV2, i.e. the key of the specific piece of content, not the library it ended up in. Note on testing: Although there were unit tests for the migration of legacy content libraries, there were not any unit tests for the migration of legacy library *blocks*. This commit adds a minimal test, which would have caught the bug we're fixing. It would be good to add more comprehensive testing unit testing for this part of the migration flow. --------- Co-authored-by: Kyle McCormick --- xmodule/library_content_block.py | 14 ++++--- xmodule/tests/test_library_content.py | 53 ++++++++++++++++++++++----- 2 files changed, 53 insertions(+), 14 deletions(-) diff --git a/xmodule/library_content_block.py b/xmodule/library_content_block.py index 5853b3923f46..fd65053f7a42 100644 --- a/xmodule/library_content_block.py +++ b/xmodule/library_content_block.py @@ -17,7 +17,7 @@ import nh3 from django.core.exceptions import ObjectDoesNotExist, PermissionDenied -from opaque_keys.edx.locator import LibraryLocator, LibraryLocatorV2 +from opaque_keys.edx.locator import LibraryLocator, LibraryUsageLocatorV2 from web_fragments.fragment import Fragment from webob import Response from xblock.core import XBlock @@ -324,11 +324,15 @@ def _v2_update_children_upstream_version(self): store = modulestore() with store.bulk_operations(self.course_id): children = self.get_children() - child_migrations = migrator_api.get_forwarding_for_blocks([child.usage_key for child in children]) - for child in children: - old_upstream_key, _ = self.runtime.modulestore.get_block_original_usage(child.usage_key) + # These are the v1 library item upstream UsageKeys + child_old_upstream_keys = [ + self.runtime.modulestore.get_block_original_usage(child.usage_key)[0] + for child in children + ] + child_migrations = migrator_api.get_forwarding_for_blocks(child_old_upstream_keys) + for child, old_upstream_key in zip(children, child_old_upstream_keys): upstream_migration = child_migrations.get(old_upstream_key) - if upstream_migration and isinstance(upstream_migration.target_key, LibraryLocatorV2): + if upstream_migration and isinstance(upstream_migration.target_key, LibraryUsageLocatorV2): child.upstream = str(upstream_migration.target_key) if upstream_migration.target_version_num: child.upstream_version = upstream_migration.target_version_num diff --git a/xmodule/tests/test_library_content.py b/xmodule/tests/test_library_content.py index 0f7807d61bc8..d1b0d4422b50 100644 --- a/xmodule/tests/test_library_content.py +++ b/xmodule/tests/test_library_content.py @@ -736,9 +736,9 @@ def test_removed_invalid(self): ) @patch('xmodule.html_block.HtmlBlock.author_view', dummy_render, create=True) @patch('xmodule.x_module.ModuleStoreRuntime.applicable_aside_types', lambda self, block: []) -class TestMigratedLibraryContentRender(LegacyLibraryContentTest): +class TestLegacyLibraryContentBlockMigration(LegacyLibraryContentTest): """ - Rendering unit tests for LegacyLibraryContentBlock + Unit tests for LegacyLibraryContentBlock """ def setUp(self): @@ -747,16 +747,14 @@ def setUp(self): super().setUp() user = UserFactory() self._sync_lc_block_from_library() - self.organization = OrganizationFactory() - self.lib_key_v2 = LibraryLocatorV2.from_string( - f"lib:{self.organization.short_name}:test-key" - ) + self.organization = OrganizationFactory(short_name="myorg") + self.lib_key_v2 = LibraryLocatorV2.from_string("lib:myorg:mylib") lib_api.create_library( org=self.organization, - slug=self.lib_key_v2.slug, - title="Test Library", + slug="mylib", + title="My Test V2 Library", ) - self.library_v2 = lib_api.ContentLibrary.objects.get(slug=self.lib_key_v2.slug) + self.library_v2 = lib_api.ContentLibrary.objects.get(slug="mylib") api.start_migration_to_library( user=user, source_key=self.library.location.library_key, @@ -770,6 +768,43 @@ def setUp(self): # Migrate block self.lc_block.upgrade_to_v2_library(None, None) + def test_migration_of_fields(self): + """ + Test that the LC block migration correctly updates the metadata of the LC block and its children. + + This tests only the simplest state: The source lib has been migrated with forwarding, exactly once, + and the LC block has also been migrated. + + TODO(https://github.com/openedx/edx-platform/issues/37837): + It would be good to also test more cases, including: + * When migration occurs which is non-forwarding, it does *not* affect the childen of this block. + * When the library migration HAS happend but the LC block migration HASN'T YET, then the fields of + the block and its children will be unchanged, but the user will be prompted to upgrade. + * When some or all of the blocks already exist in the target library before the migration, then + the migration target versions will NOT all be 1, and the upstream_versions should reflect that. + * When the target library blocks have been edited and published AFTER the legacy library migration + but BEFORE the LC block migration, then executing the LC block migration will set upstream_version + based on the migration target versions, NOT the latest versions. + """ + assert self.lc_block.is_migrated_to_v2 is True + children = self.lc_block.get_children() + assert len(children) == len(self.lib_blocks) + # The children's legacy library blocks have been migrated to a V2 library. + # We expect that each child's `upstream` has been updated to point at + # the target of each library block's migration. + assert children[0].upstream == "lb:myorg:mylib:html:html_1" + assert children[1].upstream == "lb:myorg:mylib:html:html_2" + assert children[2].upstream == "lb:myorg:mylib:html:html_3" + assert children[3].upstream == "lb:myorg:mylib:html:html_4" + # We also expect that each child's `upstream_version` has been set to the + # version of the migrated library block at the time of its migration, which + # we are assuming is `1` (i.e., the first version, as the blocks did not + # previously exist in the target library). + assert children[0].upstream_version == 1 + assert children[1].upstream_version == 1 + assert children[2].upstream_version == 1 + assert children[3].upstream_version == 1 + def test_preview_view(self): """ Test preview view rendering """ assert len(self.lc_block.children) == len(self.lib_blocks) From 1e8714f6c0b27f3bfc50ab33a317bc860846d978 Mon Sep 17 00:00:00 2001 From: Chintan Joshi Date: Tue, 6 Jan 2026 19:01:33 +0530 Subject: [PATCH 161/351] feat: soft delete (#37) Implements soft delete functionality for discussion threads, responses, and comments using the is_deleted flag instead of permanently deleting records. This enables safe deletion and restoration of discussion content while preserving existing data. --- lms/djangoapps/discussion/rest_api/api.py | 1272 ++++++++++++---- lms/djangoapps/discussion/rest_api/forms.py | 73 +- .../discussion/rest_api/serializers.py | 287 +++- lms/djangoapps/discussion/rest_api/tasks.py | 110 +- .../discussion/rest_api/tests/test_api_v2.py | 105 +- .../discussion/rest_api/tests/test_forms.py | 99 +- .../rest_api/tests/test_serializers.py | 592 ++++---- .../discussion/rest_api/tests/test_views.py | 1284 ++++++++++------- .../rest_api/tests/test_views_v2.py | 65 +- .../discussion/rest_api/tests/utils.py | 440 +++--- lms/djangoapps/discussion/rest_api/urls.py | 64 +- lms/djangoapps/discussion/rest_api/views.py | 583 ++++++-- .../comment_client/comment.py | 142 +- .../comment_client/models.py | 96 +- .../comment_client/thread.py | 325 +++-- 15 files changed, 3737 insertions(+), 1800 deletions(-) diff --git a/lms/djangoapps/discussion/rest_api/api.py b/lms/djangoapps/discussion/rest_api/api.py index b87852c16cfa..fcc13efc40b8 100644 --- a/lms/djangoapps/discussion/rest_api/api.py +++ b/lms/djangoapps/discussion/rest_api/api.py @@ -1,17 +1,17 @@ """ Discussion API internal interface """ + from __future__ import annotations import itertools +import logging import re from collections import defaultdict from datetime import datetime - from enum import Enum from typing import Dict, Iterable, List, Literal, Optional, Set, Tuple from urllib.parse import urlencode, urlunparse -from pytz import UTC from django.conf import settings from django.contrib.auth import get_user_model @@ -19,24 +19,26 @@ from django.db.models import Q from django.http import Http404 from django.urls import reverse +from django.utils.html import strip_tags from edx_django_utils.monitoring import function_trace from opaque_keys import InvalidKeyError from opaque_keys.edx.locator import CourseKey +from pytz import UTC from rest_framework import status from rest_framework.exceptions import PermissionDenied from rest_framework.request import Request from rest_framework.response import Response -from common.djangoapps.student.roles import ( - CourseInstructorRole, - CourseStaffRole, -) - +from common.djangoapps.student.roles import CourseInstructorRole, CourseStaffRole +from forum import api as forum_api from lms.djangoapps.course_api.blocks.api import get_blocks from lms.djangoapps.courseware.courses import get_course_with_access from lms.djangoapps.courseware.exceptions import CourseAccessRedirect from lms.djangoapps.discussion.rate_limit import is_content_creation_rate_limited -from lms.djangoapps.discussion.toggles import ENABLE_DISCUSSIONS_MFE, ONLY_VERIFIED_USERS_CAN_POST +from lms.djangoapps.discussion.toggles import ( + ENABLE_DISCUSSIONS_MFE, + ONLY_VERIFIED_USERS_CAN_POST, +) from lms.djangoapps.discussion.views import is_privileged_user from openedx.core.djangoapps.discussions.models import ( DiscussionsConfiguration, @@ -48,12 +50,12 @@ from openedx.core.djangoapps.django_comment_common.comment_client.comment import Comment from openedx.core.djangoapps.django_comment_common.comment_client.course import ( get_course_commentable_counts, - get_course_user_stats + get_course_user_stats, ) from openedx.core.djangoapps.django_comment_common.comment_client.thread import Thread from openedx.core.djangoapps.django_comment_common.comment_client.utils import ( CommentClient500Error, - CommentClientRequestError + CommentClientRequestError, ) from openedx.core.djangoapps.django_comment_common.models import ( FORUM_ROLE_ADMINISTRATOR, @@ -61,13 +63,13 @@ FORUM_ROLE_GROUP_MODERATOR, FORUM_ROLE_MODERATOR, CourseDiscussionSettings, - Role + Role, ) from openedx.core.djangoapps.django_comment_common.signals import ( comment_created, comment_deleted, - comment_endorsed, comment_edited, + comment_endorsed, comment_flagged, comment_voted, thread_created, @@ -75,11 +77,15 @@ thread_edited, thread_flagged, thread_followed, + thread_unfollowed, thread_voted, - thread_unfollowed ) from openedx.core.djangoapps.user_api.accounts.api import get_account_settings -from openedx.core.lib.exceptions import CourseNotFoundError, DiscussionNotFoundError, PageNotFoundError +from openedx.core.lib.exceptions import ( + CourseNotFoundError, + DiscussionNotFoundError, + PageNotFoundError, +) from xmodule.course_block import CourseBlock from xmodule.modulestore import ModuleStoreEnum from xmodule.modulestore.django import modulestore @@ -88,21 +94,27 @@ from ..django_comment_client.base.views import ( track_comment_created_event, track_comment_deleted_event, + track_discussion_reported_event, + track_discussion_unreported_event, + track_forum_search_event, track_thread_created_event, track_thread_deleted_event, + track_thread_followed_event, track_thread_viewed_event, track_voted_event, - track_discussion_reported_event, - track_discussion_unreported_event, - track_forum_search_event, track_thread_followed_event ) from ..django_comment_client.utils import ( get_group_id_for_user, get_user_role_names, has_discussion_privileges, - is_commentable_divided + is_commentable_divided, +) +from .exceptions import ( + CommentNotFoundError, + DiscussionBlackOutException, + DiscussionDisabledError, + ThreadNotFoundError, ) -from .exceptions import CommentNotFoundError, DiscussionBlackOutException, DiscussionDisabledError, ThreadNotFoundError from .forms import CommentActionsForm, ThreadActionsForm, UserOrdering from .pagination import DiscussionAPIPagination from .permissions import ( @@ -110,7 +122,7 @@ can_take_action_on_spam, get_editable_fields, get_initializable_comment_fields, - get_initializable_thread_fields + get_initializable_thread_fields, ) from .serializers import ( CommentSerializer, @@ -119,20 +131,23 @@ ThreadSerializer, TopicOrdering, UserStatsSerializer, - get_context + get_context, ) from .utils import ( AttributeDict, add_stats_for_users_with_no_discussion_content, + can_user_notify_all_learners, create_blocks_params, discussion_open_for_user, + get_captcha_site_key_by_platform, get_usernames_for_course, get_usernames_from_search_string, - set_attribute, + is_captcha_enabled, is_posting_allowed, - can_user_notify_all_learners, is_captcha_enabled, get_captcha_site_key_by_platform + set_attribute, ) +log = logging.getLogger(__name__) User = get_user_model() ThreadType = Literal["discussion", "question"] @@ -166,11 +181,14 @@ class DiscussionEntity(Enum): """ Enum for different types of discussion related entities """ - thread = 'thread' - comment = 'comment' + + thread = "thread" + comment = "comment" -def _get_course(course_key: CourseKey, user: User, check_tab: bool = True) -> CourseBlock: +def _get_course( + course_key: CourseKey, user: User, check_tab: bool = True +) -> CourseBlock: """ Get the course block, raising CourseNotFoundError if the course is not found or the user cannot access forums for the course, and DiscussionDisabledError if the @@ -188,14 +206,16 @@ def _get_course(course_key: CourseKey, user: User, check_tab: bool = True) -> Co CourseBlock: course object """ try: - course = get_course_with_access(user, 'load', course_key, check_if_enrolled=True) + course = get_course_with_access( + user, "load", course_key, check_if_enrolled=True + ) except (Http404, CourseAccessRedirect) as err: # Convert 404s into CourseNotFoundErrors. # Raise course not found if the user cannot access the course raise CourseNotFoundError("Course not found.") from err if check_tab: - discussion_tab = CourseTabList.get_tab_by_type(course.tabs, 'discussion') + discussion_tab = CourseTabList.get_tab_by_type(course.tabs, "discussion") if not (discussion_tab and discussion_tab.is_enabled(course, user)): raise DiscussionDisabledError("Discussion is disabled for the course.") @@ -216,22 +236,34 @@ def _get_thread_and_context(request, thread_id, retrieve_kwargs=None, course_id= retrieve_kwargs["with_responses"] = False if "mark_as_read" not in retrieve_kwargs: retrieve_kwargs["mark_as_read"] = False - cc_thread = Thread(id=thread_id).retrieve(course_id=course_id, **retrieve_kwargs) + cc_thread = Thread(id=thread_id).retrieve( + course_id=course_id, **retrieve_kwargs + ) course_key = CourseKey.from_string(cc_thread["course_id"]) course = _get_course(course_key, request.user) context = get_context(course, request, cc_thread) - if retrieve_kwargs.get("flagged_comments") and not context["has_moderation_privilege"]: + if ( + retrieve_kwargs.get("flagged_comments") + and not context["has_moderation_privilege"] + ): raise ValidationError("Only privileged users can request flagged comments") course_discussion_settings = CourseDiscussionSettings.get(course_key) if ( - not context["has_moderation_privilege"] and - cc_thread["group_id"] and - is_commentable_divided(course.id, cc_thread["commentable_id"], course_discussion_settings) + not context["has_moderation_privilege"] + and cc_thread["group_id"] + and is_commentable_divided( + course.id, cc_thread["commentable_id"], course_discussion_settings + ) ): - requester_group_id = get_group_id_for_user(request.user, course_discussion_settings) - if requester_group_id is not None and cc_thread["group_id"] != requester_group_id: + requester_group_id = get_group_id_for_user( + request.user, course_discussion_settings + ) + if ( + requester_group_id is not None + and cc_thread["group_id"] != requester_group_id + ): raise ThreadNotFoundError("Thread not found.") return cc_thread, context except CommentClientRequestError as err: @@ -264,8 +296,8 @@ def _is_user_author_or_privileged(cc_content, context): Boolean """ return ( - context["has_moderation_privilege"] or - context["cc_requester"]["id"] == cc_content["user_id"] + context["has_moderation_privilege"] + or context["cc_requester"]["id"] == cc_content["user_id"] ) @@ -275,11 +307,13 @@ def get_thread_list_url(request, course_key, topic_id_list=None, following=False """ path = reverse("thread-list") query_list = ( - [("course_id", str(course_key))] + - [("topic_id", topic_id) for topic_id in topic_id_list or []] + - ([("following", following)] if following else []) + [("course_id", str(course_key))] + + [("topic_id", topic_id) for topic_id in topic_id_list or []] + + ([("following", following)] if following else []) + ) + return request.build_absolute_uri( + urlunparse(("", "", path, "", urlencode(query_list), "")) ) - return request.build_absolute_uri(urlunparse(("", "", path, "", urlencode(query_list), ""))) def get_course(request, course_key, check_tab=True): @@ -324,18 +358,19 @@ def _format_datetime(dt): the substitution... though really, that would probably break mobile client parsing of the dates as well. :-P """ - return dt.isoformat().replace('+00:00', 'Z') + return dt.isoformat().replace("+00:00", "Z") course = _get_course(course_key, request.user, check_tab=check_tab) user_roles = get_user_role_names(request.user, course_key) course_config = DiscussionsConfiguration.get(course_key) EDIT_REASON_CODES = getattr(settings, "DISCUSSION_MODERATION_EDIT_REASON_CODES", {}) - CLOSE_REASON_CODES = getattr(settings, "DISCUSSION_MODERATION_CLOSE_REASON_CODES", {}) + CLOSE_REASON_CODES = getattr( + settings, "DISCUSSION_MODERATION_CLOSE_REASON_CODES", {} + ) is_posting_enabled = is_posting_allowed( - course_config.posting_restrictions, - course.get_discussion_blackout_datetimes() + course_config.posting_restrictions, course.get_discussion_blackout_datetimes() ) - discussion_tab = CourseTabList.get_tab_by_type(course.tabs, 'discussion') + discussion_tab = CourseTabList.get_tab_by_type(course.tabs, "discussion") is_course_staff = CourseStaffRole(course_key).has_user(request.user) is_course_admin = CourseInstructorRole(course_key).has_user(request.user) return { @@ -349,7 +384,9 @@ def _format_datetime(dt): for blackout in course.get_discussion_blackout_datetimes() ], "thread_list_url": get_thread_list_url(request, course_key), - "following_thread_list_url": get_thread_list_url(request, course_key, following=True), + "following_thread_list_url": get_thread_list_url( + request, course_key, following=True + ), "topics_url": request.build_absolute_uri( reverse("course_topics", kwargs={"course_id": course_key}) ), @@ -357,18 +394,23 @@ def _format_datetime(dt): "allow_anonymous_to_peers": course.allow_anonymous_to_peers, "user_roles": user_roles, "has_bulk_delete_privileges": can_take_action_on_spam(request.user, course_key), - "has_moderation_privileges": bool(user_roles & { - FORUM_ROLE_ADMINISTRATOR, - FORUM_ROLE_MODERATOR, - FORUM_ROLE_COMMUNITY_TA, - }), + "has_moderation_privileges": bool( + user_roles + & { + FORUM_ROLE_ADMINISTRATOR, + FORUM_ROLE_MODERATOR, + FORUM_ROLE_COMMUNITY_TA, + } + ), "is_group_ta": bool(user_roles & {FORUM_ROLE_GROUP_MODERATOR}), "is_user_admin": request.user.is_staff, "is_course_staff": is_course_staff, "is_course_admin": is_course_admin, "provider": course_config.provider_type, "enable_in_context": course_config.enable_in_context, - "group_at_subsection": course_config.plugin_configuration.get("group_at_subsection", False), + "group_at_subsection": course_config.plugin_configuration.get( + "group_at_subsection", False + ), "edit_reasons": [ {"code": reason_code, "label": label} for (reason_code, label) in EDIT_REASON_CODES.items() @@ -377,17 +419,23 @@ def _format_datetime(dt): {"code": reason_code, "label": label} for (reason_code, label) in CLOSE_REASON_CODES.items() ], - 'show_discussions': bool(discussion_tab and discussion_tab.is_enabled(course, request.user)), - 'is_notify_all_learners_enabled': can_user_notify_all_learners( + "show_discussions": bool( + discussion_tab and discussion_tab.is_enabled(course, request.user) + ), + "is_notify_all_learners_enabled": can_user_notify_all_learners( user_roles, is_course_staff, is_course_admin ), - 'captcha_settings': { - 'enabled': is_captcha_enabled(course_key), - 'site_key': get_captcha_site_key_by_platform('web'), + "captcha_settings": { + "enabled": is_captcha_enabled(course_key), + "site_key": get_captcha_site_key_by_platform("web"), }, "is_email_verified": request.user.is_active, - "only_verified_users_can_post": ONLY_VERIFIED_USERS_CAN_POST.is_enabled(course_key), - "content_creation_rate_limited": is_content_creation_rate_limited(request, course_key, increment=False), + "only_verified_users_can_post": ONLY_VERIFIED_USERS_CAN_POST.is_enabled( + course_key + ), + "content_creation_rate_limited": is_content_creation_rate_limited( + request, course_key, increment=False + ), } @@ -440,7 +488,7 @@ def convert(text): return text def alphanum_key(key): - return [convert(c) for c in re.split('([0-9]+)', key)] + return [convert(c) for c in re.split("([0-9]+)", key)] return sorted(category_list, key=alphanum_key) @@ -482,7 +530,7 @@ def get_non_courseware_topics( course_key: CourseKey, course: CourseBlock, topic_ids: Optional[List[str]], - thread_counts: Dict[str, Dict[str, int]] + thread_counts: Dict[str, Dict[str, int]], ) -> Tuple[List[Dict], Set[str]]: """ Returns a list of topic trees that are not linked to courseware. @@ -506,13 +554,17 @@ def get_non_courseware_topics( existing_topic_ids = set() topics = list(course.discussion_topics.items()) for name, entry in topics: - if not topic_ids or entry['id'] in topic_ids: + if not topic_ids or entry["id"] in topic_ids: discussion_topic = DiscussionTopic( - entry["id"], name, get_thread_list_url(request, course_key, [entry["id"]]), + entry["id"], + name, + get_thread_list_url(request, course_key, [entry["id"]]), None, - thread_counts.get(entry["id"]) + thread_counts.get(entry["id"]), + ) + non_courseware_topics.append( + DiscussionTopicSerializer(discussion_topic).data ) - non_courseware_topics.append(DiscussionTopicSerializer(discussion_topic).data) if topic_ids and entry["id"] in topic_ids: existing_topic_ids.add(entry["id"]) @@ -520,7 +572,9 @@ def get_non_courseware_topics( return non_courseware_topics, existing_topic_ids -def get_course_topics(request: Request, course_key: CourseKey, topic_ids: Optional[Set[str]] = None): +def get_course_topics( + request: Request, course_key: CourseKey, topic_ids: Optional[Set[str]] = None +): """ Returns the course topic listing for the given course and user; filtered by 'topic_ids' list if given. @@ -544,15 +598,25 @@ def get_course_topics(request: Request, course_key: CourseKey, topic_ids: Option courseware_topics, existing_courseware_topic_ids = get_courseware_topics( request, course_key, course, topic_ids, thread_counts ) - non_courseware_topics, existing_non_courseware_topic_ids = get_non_courseware_topics( - request, course_key, course, topic_ids, thread_counts, + non_courseware_topics, existing_non_courseware_topic_ids = ( + get_non_courseware_topics( + request, + course_key, + course, + topic_ids, + thread_counts, + ) ) if topic_ids: - not_found_topic_ids = topic_ids - (existing_courseware_topic_ids | existing_non_courseware_topic_ids) + not_found_topic_ids = topic_ids - ( + existing_courseware_topic_ids | existing_non_courseware_topic_ids + ) if not_found_topic_ids: raise DiscussionNotFoundError( - "Discussion not found for '{}'.".format(", ".join(str(id) for id in not_found_topic_ids)) + "Discussion not found for '{}'.".format( + ", ".join(str(id) for id in not_found_topic_ids) + ) ) return { @@ -567,17 +631,19 @@ def get_v2_non_courseware_topics_as_v1(request, course_key, topics): """ non_courseware_topics = [] for topic in topics: - if topic.get('usage_key', '') is None: - for key in ['usage_key', 'enabled_in_context']: + if topic.get("usage_key", "") is None: + for key in ["usage_key", "enabled_in_context"]: topic.pop(key) - topic.update({ - 'children': [], - 'thread_list_url': get_thread_list_url( - request, - course_key, - topic.get('id'), - ) - }) + topic.update( + { + "children": [], + "thread_list_url": get_thread_list_url( + request, + course_key, + topic.get("id"), + ), + } + ) non_courseware_topics.append(topic) return non_courseware_topics @@ -589,23 +655,25 @@ def get_v2_courseware_topics_as_v1(request, course_key, sequentials, topics): courseware_topics = [] for sequential in sequentials: children = [] - for child in sequential.get('children', []): + for child in sequential.get("children", []): for topic in topics: - if child == topic.get('usage_key'): - topic.update({ - 'children': [], - 'thread_list_url': get_thread_list_url( - request, - course_key, - [topic.get('id')], - ) - }) - topic.pop('enabled_in_context') + if child == topic.get("usage_key"): + topic.update( + { + "children": [], + "thread_list_url": get_thread_list_url( + request, + course_key, + [topic.get("id")], + ), + } + ) + topic.pop("enabled_in_context") children.append(AttributeDict(topic)) discussion_topic = DiscussionTopic( None, - sequential.get('display_name'), + sequential.get("display_name"), get_thread_list_url( request, course_key, @@ -618,7 +686,7 @@ def get_v2_courseware_topics_as_v1(request, course_key, sequentials, topics): courseware_topics = [ courseware_topic for courseware_topic in courseware_topics - if courseware_topic.get('children', []) + if courseware_topic.get("children", []) ] return courseware_topics @@ -635,20 +703,21 @@ def get_v2_course_topics_as_v1( blocks_params = create_blocks_params(course_usage_key, request.user) blocks = get_blocks( request, - blocks_params['usage_key'], - blocks_params['user'], - blocks_params['depth'], - blocks_params['nav_depth'], - blocks_params['requested_fields'], - blocks_params['block_counts'], - blocks_params['student_view_data'], - blocks_params['return_type'], - blocks_params['block_types_filter'], + blocks_params["usage_key"], + blocks_params["user"], + blocks_params["depth"], + blocks_params["nav_depth"], + blocks_params["requested_fields"], + blocks_params["block_counts"], + blocks_params["student_view_data"], + blocks_params["return_type"], + blocks_params["block_types_filter"], hide_access_denials=False, - )['blocks'] + )["blocks"] - sequentials = [value for _, value in blocks.items() - if value.get('type') == "sequential"] + sequentials = [ + value for _, value in blocks.items() if value.get("type") == "sequential" + ] topics = get_course_topics_v2(course_key, request.user, topic_ids) non_courseware_topics = get_v2_non_courseware_topics_as_v1( @@ -705,24 +774,29 @@ def get_course_topics_v2( # Check access to the course store = modulestore() _get_course(course_key, user=user, check_tab=False) - user_is_privileged = user.is_staff or user.roles.filter( - course_id=course_key, - name__in=[ - FORUM_ROLE_MODERATOR, - FORUM_ROLE_COMMUNITY_TA, - FORUM_ROLE_ADMINISTRATOR, - ] - ).exists() + user_is_privileged = ( + user.is_staff + or user.roles.filter( + course_id=course_key, + name__in=[ + FORUM_ROLE_MODERATOR, + FORUM_ROLE_COMMUNITY_TA, + FORUM_ROLE_ADMINISTRATOR, + ], + ).exists() + ) with store.branch_setting(ModuleStoreEnum.Branch.draft_preferred, course_key): blocks = store.get_items( course_key, - qualifiers={'category': 'vertical'}, - fields=['usage_key', 'discussion_enabled', 'display_name'], + qualifiers={"category": "vertical"}, + fields=["usage_key", "discussion_enabled", "display_name"], ) accessible_vertical_keys = [] for block in blocks: - if block.discussion_enabled and (not block.visible_to_staff_only or user_is_privileged): + if block.discussion_enabled and ( + not block.visible_to_staff_only or user_is_privileged + ): accessible_vertical_keys.append(block.usage_key) accessible_vertical_keys.append(None) @@ -732,9 +806,13 @@ def get_course_topics_v2( ) if user_is_privileged: - topics_query = topics_query.filter(Q(usage_key__in=accessible_vertical_keys) | Q(enabled_in_context=False)) + topics_query = topics_query.filter( + Q(usage_key__in=accessible_vertical_keys) | Q(enabled_in_context=False) + ) else: - topics_query = topics_query.filter(usage_key__in=accessible_vertical_keys, enabled_in_context=True) + topics_query = topics_query.filter( + usage_key__in=accessible_vertical_keys, enabled_in_context=True + ) if topic_ids: topics_query = topics_query.filter(external_id__in=topic_ids) @@ -746,11 +824,13 @@ def get_course_topics_v2( reverse=True, ) elif order_by == TopicOrdering.NAME: - topics_query = topics_query.order_by('title') + topics_query = topics_query.order_by("title") else: - topics_query = topics_query.order_by('ordering') + topics_query = topics_query.order_by("ordering") - topics_data = DiscussionTopicSerializerV2(topics_query, many=True, context={"thread_counts": thread_counts}).data + topics_data = DiscussionTopicSerializerV2( + topics_query, many=True, context={"thread_counts": thread_counts} + ).data return [ topic_data for topic_data in topics_data @@ -777,7 +857,7 @@ def _get_user_profile_dict(request, usernames): else: username_list = [] user_profile_details = get_account_settings(request, username_list) - return {user['username']: user for user in user_profile_details} + return {user["username"]: user for user in user_profile_details} def _user_profile(user_profile): @@ -785,11 +865,7 @@ def _user_profile(user_profile): Returns the user profile object. For now, this just comprises the profile_image details. """ - return { - 'profile': { - 'image': user_profile['profile_image'] - } - } + return {"profile": {"image": user_profile["profile_image"]}} def _get_users(discussion_entity_type, discussion_entity, username_profile_dict): @@ -807,22 +883,28 @@ def _get_users(discussion_entity_type, discussion_entity, username_profile_dict) A dict of users with username as key and user profile details as value. """ users = {} - if discussion_entity['author']: - user_profile = username_profile_dict.get(discussion_entity['author']) + if discussion_entity["author"]: + user_profile = username_profile_dict.get(discussion_entity["author"]) if user_profile: - users[discussion_entity['author']] = _user_profile(user_profile) + users[discussion_entity["author"]] = _user_profile(user_profile) if ( discussion_entity_type == DiscussionEntity.comment - and discussion_entity['endorsed'] - and discussion_entity['endorsed_by'] + and discussion_entity["endorsed"] + and discussion_entity["endorsed_by"] ): - users[discussion_entity['endorsed_by']] = _user_profile(username_profile_dict[discussion_entity['endorsed_by']]) + users[discussion_entity["endorsed_by"]] = _user_profile( + username_profile_dict[discussion_entity["endorsed_by"]] + ) return users def _add_additional_response_fields( - request, serialized_discussion_entities, usernames, discussion_entity_type, include_profile_image + request, + serialized_discussion_entities, + usernames, + discussion_entity_type, + include_profile_image, ): """ Adds additional data to serialized discussion thread/comment. @@ -840,9 +922,13 @@ def _add_additional_response_fields( A list of serialized discussion thread/comment with additional data if requested. """ if include_profile_image: - username_profile_dict = _get_user_profile_dict(request, usernames=','.join(usernames)) + username_profile_dict = _get_user_profile_dict( + request, usernames=",".join(usernames) + ) for discussion_entity in serialized_discussion_entities: - discussion_entity['users'] = _get_users(discussion_entity_type, discussion_entity, username_profile_dict) + discussion_entity["users"] = _get_users( + discussion_entity_type, discussion_entity, username_profile_dict + ) return serialized_discussion_entities @@ -851,10 +937,12 @@ def _include_profile_image(requested_fields): """ Returns True if requested_fields list has 'profile_image' entity else False """ - return requested_fields and 'profile_image' in requested_fields + return requested_fields and "profile_image" in requested_fields -def _serialize_discussion_entities(request, context, discussion_entities, requested_fields, discussion_entity_type): +def _serialize_discussion_entities( + request, context, discussion_entities, requested_fields, discussion_entity_type +): """ It serializes Discussion Entity (Thread or Comment) and add additional data if requested. @@ -885,14 +973,19 @@ def _serialize_discussion_entities(request, context, discussion_entities, reques results.append(serialized_entity) if include_profile_image: - if serialized_entity['author'] and serialized_entity['author'] not in usernames: - usernames.append(serialized_entity['author']) if ( - 'endorsed' in serialized_entity and serialized_entity['endorsed'] and - 'endorsed_by' in serialized_entity and - serialized_entity['endorsed_by'] and serialized_entity['endorsed_by'] not in usernames + serialized_entity["author"] + and serialized_entity["author"] not in usernames ): - usernames.append(serialized_entity['endorsed_by']) + usernames.append(serialized_entity["author"]) + if ( + "endorsed" in serialized_entity + and serialized_entity["endorsed"] + and "endorsed_by" in serialized_entity + and serialized_entity["endorsed_by"] + and serialized_entity["endorsed_by"] not in usernames + ): + usernames.append(serialized_entity["endorsed_by"]) results = _add_additional_response_fields( request, results, usernames, discussion_entity_type, include_profile_image @@ -916,6 +1009,7 @@ def get_thread_list( order_direction: Literal["desc"] = "desc", requested_fields: Optional[List[Literal["profile_image"]]] = None, count_flagged: bool = None, + show_deleted: bool = False, ): """ Return the list of all discussion threads pertaining to the given course @@ -959,20 +1053,31 @@ def get_thread_list( CourseNotFoundError: if the requesting user does not have access to the requested course PageNotFoundError: if page requested is beyond the last """ - exclusive_param_count = sum(1 for param in [topic_id_list, text_search, following] if param) + exclusive_param_count = sum( + 1 for param in [topic_id_list, text_search, following] if param + ) if exclusive_param_count > 1: # pragma: no cover - raise ValueError("More than one mutually exclusive param passed to get_thread_list") + raise ValueError( + "More than one mutually exclusive param passed to get_thread_list" + ) - cc_map = {"last_activity_at": "activity", "comment_count": "comments", "vote_count": "votes"} + cc_map = { + "last_activity_at": "activity", + "comment_count": "comments", + "vote_count": "votes", + } if order_by not in cc_map: - raise ValidationError({ - "order_by": - [f"Invalid value. '{order_by}' must be 'last_activity_at', 'comment_count', or 'vote_count'"] - }) + raise ValidationError( + { + "order_by": [ + f"Invalid value. '{order_by}' must be 'last_activity_at', 'comment_count', or 'vote_count'" + ] + } + ) if order_direction != "desc": - raise ValidationError({ - "order_direction": [f"Invalid value. '{order_direction}' must be 'desc'"] - }) + raise ValidationError( + {"order_direction": [f"Invalid value. '{order_direction}' must be 'desc'"]} + ) course = _get_course(course_key, request.user) context = get_context(course, request) @@ -984,13 +1089,21 @@ def get_thread_list( except User.DoesNotExist: # Raising an error for a missing user leaks the presence of a username, # so just return an empty response. - return DiscussionAPIPagination(request, 0, 1).get_paginated_response({ - "results": [], - "text_search_rewrite": None, - }) + return DiscussionAPIPagination(request, 0, 1).get_paginated_response( + { + "results": [], + "text_search_rewrite": None, + } + ) if count_flagged and not context["has_moderation_privilege"]: - raise PermissionDenied("`count_flagged` can only be set by users with moderator access or higher.") + raise PermissionDenied( + "`count_flagged` can only be set by users with moderator access or higher." + ) + if show_deleted and not context["has_moderation_privilege"]: + raise PermissionDenied( + "`show_deleted` can only be set by users with moderator access or higher." + ) group_id = None allowed_roles = [ @@ -1010,7 +1123,9 @@ def get_thread_list( not context["has_moderation_privilege"] or request.user.id in context["ta_user_ids"] ): - group_id = get_group_id_for_user(request.user, CourseDiscussionSettings.get(course.id)) + group_id = get_group_id_for_user( + request.user, CourseDiscussionSettings.get(course.id) + ) query_params = { "user_id": str(request.user.id), @@ -1023,21 +1138,24 @@ def get_thread_list( "flagged": flagged, "thread_type": thread_type, "count_flagged": count_flagged, + "show_deleted": show_deleted, } if view: if view in ["unread", "unanswered", "unresponded"]: query_params[view] = "true" else: - raise ValidationError({ - "view": [f"Invalid value. '{view}' must be 'unread' or 'unanswered'"] - }) + raise ValidationError( + {"view": [f"Invalid value. '{view}' must be 'unread' or 'unanswered'"]} + ) if following: paginated_results = context["cc_requester"].subscribed_threads(query_params) else: query_params["course_id"] = str(course.id) - query_params["commentable_ids"] = ",".join(topic_id_list) if topic_id_list else None + query_params["commentable_ids"] = ( + ",".join(topic_id_list) if topic_id_list else None + ) query_params["text"] = text_search paginated_results = Thread.search(query_params) # The comments service returns the last page of results if the requested @@ -1047,19 +1165,25 @@ def get_thread_list( raise PageNotFoundError("Page not found (No results on this page).") results = _serialize_discussion_entities( - request, context, paginated_results.collection, requested_fields, DiscussionEntity.thread + request, + context, + paginated_results.collection, + requested_fields, + DiscussionEntity.thread, ) paginator = DiscussionAPIPagination( request, paginated_results.page, paginated_results.num_pages, - paginated_results.thread_count + paginated_results.thread_count, + ) + return paginator.get_paginated_response( + { + "results": results, + "text_search_rewrite": paginated_results.corrected_text, + } ) - return paginator.get_paginated_response({ - "results": results, - "text_search_rewrite": paginated_results.corrected_text, - }) def get_learner_active_thread_list(request, course_key, query_params): @@ -1154,49 +1278,101 @@ def get_learner_active_thread_list(request, course_key, query_params): course = _get_course(course_key, request.user) context = get_context(course, request) - group_id = query_params.get('group_id', None) - user_id = query_params.get('user_id', None) - count_flagged = query_params.get('count_flagged', None) + group_id = query_params.get("group_id", None) + user_id = query_params.get("user_id", None) + count_flagged = query_params.get("count_flagged", None) + show_deleted = query_params.get("show_deleted", False) + if isinstance(show_deleted, str): + show_deleted = show_deleted.lower() == "true" + if user_id is None: - return Response({'detail': 'Invalid user id'}, status=status.HTTP_400_BAD_REQUEST) + return Response( + {"detail": "Invalid user id"}, status=status.HTTP_400_BAD_REQUEST + ) if count_flagged and not context["has_moderation_privilege"]: - raise PermissionDenied("count_flagged can only be set by users with moderation roles.") + raise PermissionDenied( + "count_flagged can only be set by users with moderation roles." + ) if "flagged" in query_params.keys() and not context["has_moderation_privilege"]: raise PermissionDenied("Flagged filter is only available for moderators") + if show_deleted and not context["has_moderation_privilege"]: + raise PermissionDenied( + "show_deleted can only be set by users with moderation roles." + ) if group_id is None: comment_client_user = comment_client.User(id=user_id, course_id=course_key) else: - comment_client_user = comment_client.User(id=user_id, course_id=course_key, group_id=group_id) + comment_client_user = comment_client.User( + id=user_id, course_id=course_key, group_id=group_id + ) try: threads, page, num_pages = comment_client_user.active_threads(query_params) threads = set_attribute(threads, "pinned", False) + + # This portion below is temporary until we migrate to forum v2 + filtered_threads = [] + for thread in threads: + try: + forum_thread = forum_api.get_thread( + thread.get("id"), course_id=str(course_key) + ) + is_deleted = forum_thread.get("is_deleted", False) + + if show_deleted and is_deleted: + thread["is_deleted"] = True + thread["deleted_at"] = forum_thread.get("deleted_at") + thread["deleted_by"] = forum_thread.get("deleted_by") + filtered_threads.append(thread) + elif not show_deleted and not is_deleted: + filtered_threads.append(thread) + except Exception as e: # pylint: disable=broad-exception-caught + log.warning( + "Failed to check thread %s deletion status: %s", thread.get("id"), e + ) + if not show_deleted: # Fail safe: include thread for regular users + filtered_threads.append(thread) + results = _serialize_discussion_entities( - request, context, threads, {'profile_image'}, DiscussionEntity.thread + request, + context, + filtered_threads, + {"profile_image"}, + DiscussionEntity.thread, ) paginator = DiscussionAPIPagination( - request, - page, - num_pages, - len(threads) + request, page, num_pages, len(filtered_threads) + ) + return paginator.get_paginated_response( + { + "results": results, + } ) - return paginator.get_paginated_response({ - "results": results, - }) except CommentClient500Error: return DiscussionAPIPagination( request, page_num=1, num_pages=0, - ).get_paginated_response({ - "results": [], - }) + ).get_paginated_response( + { + "results": [], + } + ) -def get_comment_list(request, thread_id, endorsed, page, page_size, flagged=False, requested_fields=None, - merge_question_type_responses=False): +def get_comment_list( + request, + thread_id, + endorsed, + page, + page_size, + flagged=False, + requested_fields=None, + merge_question_type_responses=False, + show_deleted=False, +): """ Return the list of comments in the given thread. @@ -1226,7 +1402,7 @@ def get_comment_list(request, thread_id, endorsed, page, page_size, flagged=Fals discussion.rest_api.views.CommentViewSet for more detail. """ response_skip = page_size * (page - 1) - reverse_order = request.GET.get('reverse_order', False) + reverse_order = request.GET.get("reverse_order", False) from_mfe_sidebar = request.GET.get("enable_in_context_sidebar", False) cc_thread, context = _get_thread_and_context( request, @@ -1239,19 +1415,23 @@ def get_comment_list(request, thread_id, endorsed, page, page_size, flagged=Fals "response_skip": response_skip, "response_limit": page_size, "reverse_order": reverse_order, - "merge_question_type_responses": merge_question_type_responses - } + "merge_question_type_responses": merge_question_type_responses, + }, ) # Responses to discussion threads cannot be separated by endorsed, but # responses to question threads must be separated by endorsed due to the # existing comments service interface if cc_thread["thread_type"] == "question" and not merge_question_type_responses: if endorsed is None: # lint-amnesty, pylint: disable=no-else-raise - raise ValidationError({"endorsed": ["This field is required for question threads."]}) + raise ValidationError( + {"endorsed": ["This field is required for question threads."]} + ) elif endorsed: # CS does not apply resp_skip and resp_limit to endorsed responses # of a question post - responses = cc_thread["endorsed_responses"][response_skip:(response_skip + page_size)] + responses = cc_thread["endorsed_responses"][ + response_skip: (response_skip + page_size) + ] resp_total = len(cc_thread["endorsed_responses"]) else: responses = cc_thread["non_endorsed_responses"] @@ -1260,7 +1440,11 @@ def get_comment_list(request, thread_id, endorsed, page, page_size, flagged=Fals if not merge_question_type_responses: if endorsed is not None: raise ValidationError( - {"endorsed": ["This field may not be specified for discussion threads."]} + { + "endorsed": [ + "This field may not be specified for discussion threads." + ] + } ) responses = cc_thread["children"] resp_total = cc_thread["resp_total"] @@ -1272,9 +1456,21 @@ def get_comment_list(request, thread_id, endorsed, page, page_size, flagged=Fals raise PageNotFoundError("Page not found (No results on this page).") num_pages = (resp_total + page_size - 1) // page_size if resp_total else 1 - results = _serialize_discussion_entities(request, context, responses, requested_fields, DiscussionEntity.comment) + if not show_deleted: + responses = [ + response for response in responses if not response.get("is_deleted", False) + ] + else: + if not context["has_moderation_privilege"]: + raise PermissionDenied( + "`show_deleted` can only be set by users with moderation roles." + ) + + results = _serialize_discussion_entities( + request, context, responses, requested_fields, DiscussionEntity.comment + ) - paginator = DiscussionAPIPagination(request, page, num_pages, resp_total) + paginator = DiscussionAPIPagination(request, page, num_pages, len(responses)) track_thread_viewed_event(request, context["course"], cc_thread, from_mfe_sidebar) return paginator.get_paginated_response(results) @@ -1292,7 +1488,9 @@ def _check_fields(allowed_fields, data, message): ValidationError if the given data contains a key that is not in allowed_fields """ - non_allowed_fields = {field: [message] for field in data.keys() if field not in allowed_fields} + non_allowed_fields = { + field: [message] for field in data.keys() if field not in allowed_fields + } if non_allowed_fields: raise ValidationError(non_allowed_fields) @@ -1314,7 +1512,7 @@ def _check_initializable_thread_fields(data, context): _check_fields( get_initializable_thread_fields(context), data, - "This field is not initializable." + "This field is not initializable.", ) @@ -1335,7 +1533,7 @@ def _check_initializable_comment_fields(data, context): _check_fields( get_initializable_comment_fields(context), data, - "This field is not initializable." + "This field is not initializable.", ) @@ -1345,28 +1543,40 @@ def _check_editable_fields(cc_content, data, context): editable by the requesting user """ _check_fields( - get_editable_fields(cc_content, context), - data, - "This field is not editable." + get_editable_fields(cc_content, context), data, "This field is not editable." ) -def _do_extra_actions(api_content, cc_content, request_fields, actions_form, context, request): +def _do_extra_actions( + api_content, cc_content, request_fields, actions_form, context, request +): """ Perform any necessary additional actions related to content creation or update that require a separate comments service request. """ for field, form_value in actions_form.cleaned_data.items(): - if field in request_fields and field in api_content and form_value != api_content[field]: + if ( + field in request_fields + and field in api_content + and form_value != api_content[field] + ): api_content[field] = form_value if field == "following": - _handle_following_field(form_value, context["cc_requester"], cc_content, request) + _handle_following_field( + form_value, context["cc_requester"], cc_content, request + ) elif field == "abuse_flagged": - _handle_abuse_flagged_field(form_value, context["cc_requester"], cc_content, request) + _handle_abuse_flagged_field( + form_value, context["cc_requester"], cc_content, request + ) elif field == "voted": - _handle_voted_field(form_value, cc_content, api_content, request, context) + _handle_voted_field( + form_value, cc_content, api_content, request, context + ) elif field == "read": - _handle_read_field(api_content, form_value, context["cc_requester"], cc_content) + _handle_read_field( + api_content, form_value, context["cc_requester"], cc_content + ) elif field == "pinned": _handle_pinned_field(form_value, cc_content, context["cc_requester"]) else: @@ -1376,7 +1586,7 @@ def _do_extra_actions(api_content, cc_content, request_fields, actions_form, con def _handle_following_field(form_value, user, cc_content, request): """follow/unfollow thread for the user""" course_key = CourseKey.from_string(cc_content.course_id) - course = get_course_with_access(request.user, 'load', course_key) + course = get_course_with_access(request.user, "load", course_key) if form_value: user.follow(cc_content) else: @@ -1389,15 +1599,19 @@ def _handle_following_field(form_value, user, cc_content, request): def _handle_abuse_flagged_field(form_value, user, cc_content, request): """mark or unmark thread/comment as abused""" course_key = CourseKey.from_string(cc_content.course_id) - course = get_course_with_access(request.user, 'load', course_key) + course = get_course_with_access(request.user, "load", course_key) if form_value: cc_content.flagAbuse(user, cc_content) track_discussion_reported_event(request, course, cc_content) if ENABLE_DISCUSSIONS_MFE.is_enabled(course_key): - if cc_content.type == 'thread': - thread_flagged.send(sender='flag_abuse_for_thread', user=user, post=cc_content) + if cc_content.type == "thread": + thread_flagged.send( + sender="flag_abuse_for_thread", user=user, post=cc_content + ) else: - comment_flagged.send(sender='flag_abuse_for_comment', user=user, post=cc_content) + comment_flagged.send( + sender="flag_abuse_for_comment", user=user, post=cc_content + ) else: remove_all = bool(is_privileged_user(course_key, User.objects.get(id=user.id))) cc_content.unFlagAbuse(user, cc_content, remove_all) @@ -1406,7 +1620,7 @@ def _handle_abuse_flagged_field(form_value, user, cc_content, request): def _handle_voted_field(form_value, cc_content, api_content, request, context): """vote or undo vote on thread/comment""" - signal = thread_voted if cc_content.type == 'thread' else comment_voted + signal = thread_voted if cc_content.type == "thread" else comment_voted signal.send(sender=None, user=context["request"].user, post=cc_content) if form_value: context["cc_requester"].vote(cc_content, "up") @@ -1415,7 +1629,11 @@ def _handle_voted_field(form_value, cc_content, api_content, request, context): context["cc_requester"].unvote(cc_content) api_content["vote_count"] -= 1 track_voted_event( - request, context["course"], cc_content, vote_value="up", undo_vote=not form_value + request, + context["course"], + cc_content, + vote_value="up", + undo_vote=not form_value, ) @@ -1423,7 +1641,7 @@ def _handle_read_field(api_content, form_value, user, cc_content): """ Marks thread as read for the user """ - if form_value and not cc_content['read']: + if form_value and not cc_content["read"]: user.read(cc_content) # When a thread is marked as read, all of its responses and comments # are also marked as read. @@ -1490,24 +1708,35 @@ def create_thread(request, thread_data): context = get_context(course, request) _check_initializable_thread_fields(thread_data, context) discussion_settings = CourseDiscussionSettings.get(course_key) - if ( - "group_id" not in thread_data and - is_commentable_divided(course_key, thread_data.get("topic_id"), discussion_settings) + if "group_id" not in thread_data and is_commentable_divided( + course_key, thread_data.get("topic_id"), discussion_settings ): thread_data = thread_data.copy() thread_data["group_id"] = get_group_id_for_user(user, discussion_settings) serializer = ThreadSerializer(data=thread_data, context=context) actions_form = ThreadActionsForm(thread_data) if not (serializer.is_valid() and actions_form.is_valid()): - raise ValidationError(dict(list(serializer.errors.items()) + list(actions_form.errors.items()))) + raise ValidationError( + dict(list(serializer.errors.items()) + list(actions_form.errors.items())) + ) serializer.save() cc_thread = serializer.instance - thread_created.send(sender=None, user=user, post=cc_thread, notify_all_learners=notify_all_learners) + thread_created.send( + sender=None, user=user, post=cc_thread, notify_all_learners=notify_all_learners + ) api_thread = serializer.data - _do_extra_actions(api_thread, cc_thread, list(thread_data.keys()), actions_form, context, request) + _do_extra_actions( + api_thread, cc_thread, list(thread_data.keys()), actions_form, context, request + ) - track_thread_created_event(request, course, cc_thread, actions_form.cleaned_data["following"], - from_mfe_sidebar, notify_all_learners) + track_thread_created_event( + request, + course, + cc_thread, + actions_form.cleaned_data["following"], + from_mfe_sidebar, + notify_all_learners, + ) return api_thread @@ -1546,15 +1775,30 @@ def create_comment(request, comment_data): serializer = CommentSerializer(data=comment_data, context=context) actions_form = CommentActionsForm(comment_data) if not (serializer.is_valid() and actions_form.is_valid()): - raise ValidationError(dict(list(serializer.errors.items()) + list(actions_form.errors.items()))) + raise ValidationError( + dict(list(serializer.errors.items()) + list(actions_form.errors.items())) + ) context["cc_requester"].follow(cc_thread) serializer.save() cc_comment = serializer.instance comment_created.send(sender=None, user=request.user, post=cc_comment) api_comment = serializer.data - _do_extra_actions(api_comment, cc_comment, list(comment_data.keys()), actions_form, context, request) - track_comment_created_event(request, course, cc_comment, cc_thread["commentable_id"], followed=False, - from_mfe_sidebar=from_mfe_sidebar) + _do_extra_actions( + api_comment, + cc_comment, + list(comment_data.keys()), + actions_form, + context, + request, + ) + track_comment_created_event( + request, + course, + cc_comment, + cc_thread["commentable_id"], + followed=False, + from_mfe_sidebar=from_mfe_sidebar, + ) return api_comment @@ -1576,24 +1820,32 @@ def update_thread(request, thread_id, update_data): The updated thread; see discussion.rest_api.views.ThreadViewSet for more detail. """ - cc_thread, context = _get_thread_and_context(request, thread_id, retrieve_kwargs={"with_responses": True}) + cc_thread, context = _get_thread_and_context( + request, thread_id, retrieve_kwargs={"with_responses": True} + ) _check_editable_fields(cc_thread, update_data, context) - serializer = ThreadSerializer(cc_thread, data=update_data, partial=True, context=context) + serializer = ThreadSerializer( + cc_thread, data=update_data, partial=True, context=context + ) actions_form = ThreadActionsForm(update_data) if not (serializer.is_valid() and actions_form.is_valid()): - raise ValidationError(dict(list(serializer.errors.items()) + list(actions_form.errors.items()))) + raise ValidationError( + dict(list(serializer.errors.items()) + list(actions_form.errors.items())) + ) # Only save thread object if some of the edited fields are in the thread data, not extra actions if set(update_data) - set(actions_form.fields): serializer.save() # signal to update Teams when a user edits a thread thread_edited.send(sender=None, user=request.user, post=cc_thread) api_thread = serializer.data - _do_extra_actions(api_thread, cc_thread, list(update_data.keys()), actions_form, context, request) + _do_extra_actions( + api_thread, cc_thread, list(update_data.keys()), actions_form, context, request + ) # always return read as True (and therefore unread_comment_count=0) as reasonably # accurate shortcut, rather than adding additional processing. - api_thread['read'] = True - api_thread['unread_comment_count'] = 0 + api_thread["read"] = True + api_thread["unread_comment_count"] = 0 return api_thread @@ -1628,16 +1880,27 @@ def update_comment(request, comment_id, update_data): """ cc_comment, context = _get_comment_and_context(request, comment_id) _check_editable_fields(cc_comment, update_data, context) - serializer = CommentSerializer(cc_comment, data=update_data, partial=True, context=context) + serializer = CommentSerializer( + cc_comment, data=update_data, partial=True, context=context + ) actions_form = CommentActionsForm(update_data) if not (serializer.is_valid() and actions_form.is_valid()): - raise ValidationError(dict(list(serializer.errors.items()) + list(actions_form.errors.items()))) + raise ValidationError( + dict(list(serializer.errors.items()) + list(actions_form.errors.items())) + ) # Only save comment object if some of the edited fields are in the comment data, not extra actions if set(update_data) - set(actions_form.fields): serializer.save() comment_edited.send(sender=None, user=request.user, post=cc_comment) api_comment = serializer.data - _do_extra_actions(api_comment, cc_comment, list(update_data.keys()), actions_form, context, request) + _do_extra_actions( + api_comment, + cc_comment, + list(update_data.keys()), + actions_form, + context, + request, + ) _handle_comment_signals(update_data, cc_comment, request.user) return api_comment @@ -1671,7 +1934,9 @@ def get_thread(request, thread_id, requested_fields=None, course_id=None): ) if course_id and course_id != cc_thread.course_id: raise ThreadNotFoundError("Thread not found.") - return _serialize_discussion_entities(request, context, [cc_thread], requested_fields, DiscussionEntity.thread)[0] + return _serialize_discussion_entities( + request, context, [cc_thread], requested_fields, DiscussionEntity.thread + )[0] def get_response_comments(request, comment_id, page, page_size, requested_fields=None): @@ -1699,7 +1964,10 @@ def get_response_comments(request, comment_id, page, page_size, requested_fields """ try: cc_comment = Comment(id=comment_id).retrieve() - reverse_order = request.GET.get('reverse_order', False) + reverse_order = request.GET.get("reverse_order", False) + show_deleted = request.GET.get("show_deleted", False) + show_deleted = show_deleted in ["true", "True", True] + cc_thread, context = _get_thread_and_context( request, cc_comment["thread_id"], @@ -1707,10 +1975,13 @@ def get_response_comments(request, comment_id, page, page_size, requested_fields "with_responses": True, "recursive": True, "reverse_order": reverse_order, - } + "show_deleted": show_deleted, + }, ) if cc_thread["thread_type"] == "question": - thread_responses = itertools.chain(cc_thread["endorsed_responses"], cc_thread["non_endorsed_responses"]) + thread_responses = itertools.chain( + cc_thread["endorsed_responses"], cc_thread["non_endorsed_responses"] + ) else: thread_responses = cc_thread["children"] response_comments = [] @@ -1720,16 +1991,35 @@ def get_response_comments(request, comment_id, page, page_size, requested_fields break response_skip = page_size * (page - 1) - paged_response_comments = response_comments[response_skip:(response_skip + page_size)] + paged_response_comments = response_comments[ + response_skip: (response_skip + page_size) + ] if not paged_response_comments and page != 1: raise PageNotFoundError("Page not found (No results on this page).") + if not show_deleted: + paged_response_comments = [ + response + for response in paged_response_comments + if not response.get("is_deleted", False) + ] + else: + if not context["has_moderation_privilege"]: + raise PermissionDenied( + "`show_deleted` can only be set by users with moderation roles." + ) results = _serialize_discussion_entities( - request, context, paged_response_comments, requested_fields, DiscussionEntity.comment + request, + context, + paged_response_comments, + requested_fields, + DiscussionEntity.comment, ) - comments_count = len(response_comments) - num_pages = (comments_count + page_size - 1) // page_size if comments_count else 1 + comments_count = len(paged_response_comments) + num_pages = ( + (comments_count + page_size - 1) // page_size if comments_count else 1 + ) paginator = DiscussionAPIPagination(request, page, num_pages, comments_count) return paginator.get_paginated_response(results) except CommentClientRequestError as err: @@ -1773,16 +2063,20 @@ def get_user_comments( context = get_context(course, request) if flagged and not context["has_moderation_privilege"]: - raise ValidationError("Only privileged users can filter comments by flagged status") + raise ValidationError( + "Only privileged users can filter comments by flagged status" + ) try: - response = Comment.retrieve_all({ - 'user_id': author.id, - 'course_id': str(course_key), - 'flagged': flagged, - 'page': page, - 'per_page': page_size, - }) + response = Comment.retrieve_all( + { + "user_id": author.id, + "course_id": str(course_key), + "flagged": flagged, + "page": page, + "per_page": page_size, + } + ) except CommentClientRequestError as err: raise CommentNotFoundError("Comment not found") from err @@ -1822,7 +2116,7 @@ def delete_thread(request, thread_id): """ cc_thread, context = _get_thread_and_context(request, thread_id) if can_delete(cc_thread, context): - cc_thread.delete() + cc_thread.delete(deleted_by=str(request.user.id)) thread_deleted.send(sender=None, user=request.user, post=cc_thread) track_thread_deleted_event(request, context["course"], cc_thread) else: @@ -1847,7 +2141,7 @@ def delete_comment(request, comment_id): """ cc_comment, context = _get_comment_and_context(request, comment_id) if can_delete(cc_comment, context): - cc_comment.delete() + cc_comment.delete(deleted_by=str(request.user.id)) comment_deleted.send(sender=None, user=request.user, post=cc_comment) track_comment_deleted_event(request, context["course"], cc_comment) else: @@ -1879,7 +2173,10 @@ def get_course_discussion_user_stats( """ course_key = CourseKey.from_string(course_key_str) - is_privileged = has_discussion_privileges(user=request.user, course_id=course_key) or request.user.is_staff + is_privileged = ( + has_discussion_privileges(user=request.user, course_id=course_key) + or request.user.is_staff + ) if is_privileged: order_by = order_by or UserOrdering.BY_FLAGS else: @@ -1888,30 +2185,35 @@ def get_course_discussion_user_stats( raise ValidationError({"order_by": "Invalid value"}) params = { - 'sort_key': str(order_by), - 'page': page, - 'per_page': page_size, + "sort_key": str(order_by), + "page": page, + "per_page": page_size, } comma_separated_usernames = matched_users_count = matched_users_pages = None if username_search_string: - comma_separated_usernames, matched_users_count, matched_users_pages = get_usernames_from_search_string( - course_key, username_search_string, page, page_size + comma_separated_usernames, matched_users_count, matched_users_pages = ( + get_usernames_from_search_string( + course_key, username_search_string, page, page_size + ) ) search_event_data = { - 'query': username_search_string, - 'search_type': 'Learner', - 'page': params.get('page'), - 'sort_key': params.get('sort_key'), - 'total_results': matched_users_count, + "query": username_search_string, + "search_type": "Learner", + "page": params.get("page"), + "sort_key": params.get("sort_key"), + "total_results": matched_users_count, } course = _get_course(course_key, request.user) track_forum_search_event(request, course, search_event_data) + if not comma_separated_usernames: - return DiscussionAPIPagination(request, 0, 1).get_paginated_response({ - "results": [], - }) + return DiscussionAPIPagination(request, 0, 1).get_paginated_response( + { + "results": [], + } + ) - params['usernames'] = comma_separated_usernames + params["usernames"] = comma_separated_usernames course_stats_response = get_course_user_stats(course_key, params) @@ -1931,71 +2233,429 @@ def get_course_discussion_user_stats( paginator = DiscussionAPIPagination( request, course_stats_response["page"], - matched_users_pages if username_search_string else course_stats_response["num_pages"], - matched_users_count if username_search_string else course_stats_response["count"], + ( + matched_users_pages + if username_search_string + else course_stats_response["num_pages"] + ), + ( + matched_users_count + if username_search_string + else course_stats_response["count"] + ), + ) + return paginator.get_paginated_response( + { + "results": serializer.data, + } ) - return paginator.get_paginated_response({ - "results": serializer.data, - }) def get_users_without_stats( - username_search_string, - course_key, - page_number, - page_size, - request, - is_privileged + username_search_string, course_key, page_number, page_size, request, is_privileged ): """ This return users with no user stats. This function will be deprecated when this ticket DOS-3414 is resolved """ if username_search_string: - comma_separated_usernames, matched_users_count, matched_users_pages = get_usernames_from_search_string( - course_key, username_search_string, page_number, page_size + comma_separated_usernames, matched_users_count, matched_users_pages = ( + get_usernames_from_search_string( + course_key, username_search_string, page_number, page_size + ) ) if not comma_separated_usernames: - return DiscussionAPIPagination(request, 0, 1).get_paginated_response({ - "results": [], - }) + return DiscussionAPIPagination(request, 0, 1).get_paginated_response( + { + "results": [], + } + ) else: - comma_separated_usernames, matched_users_count, matched_users_pages = get_usernames_for_course( - course_key, page_number, page_size + comma_separated_usernames, matched_users_count, matched_users_pages = ( + get_usernames_for_course(course_key, page_number, page_size) ) if comma_separated_usernames: - updated_course_stats = add_stats_for_users_with_null_values([], comma_separated_usernames) + updated_course_stats = add_stats_for_users_with_null_values( + [], comma_separated_usernames + ) - serializer = UserStatsSerializer(updated_course_stats, context={"is_privileged": is_privileged}, many=True) + serializer = UserStatsSerializer( + updated_course_stats, context={"is_privileged": is_privileged}, many=True + ) paginator = DiscussionAPIPagination( request, page_number, matched_users_pages, matched_users_count, ) - return paginator.get_paginated_response({ - "results": serializer.data, - }) + return paginator.get_paginated_response( + { + "results": serializer.data, + } + ) def add_stats_for_users_with_null_values(course_stats, users_in_course): """ Update users stats for users with no discussion stats available in course """ - users_returned_from_api = [user['username'] for user in course_stats] - user_list = users_in_course.split(',') + users_returned_from_api = [user["username"] for user in course_stats] + user_list = users_in_course.split(",") users_with_no_discussion_content = set(user_list) ^ set(users_returned_from_api) updated_course_stats = course_stats for user in users_with_no_discussion_content: - updated_course_stats.append({ - 'username': user, - 'threads': None, - 'replies': None, - 'responses': None, - 'active_flags': None, - 'inactive_flags': None, - }) - updated_course_stats = sorted(updated_course_stats, key=lambda d: len(d['username'])) + updated_course_stats.append( + { + "username": user, + "threads": None, + "replies": None, + "responses": None, + "active_flags": None, + "inactive_flags": None, + } + ) + updated_course_stats = sorted( + updated_course_stats, key=lambda d: len(d["username"]) + ) return updated_course_stats + + +def _get_user_label_function(course_staff_user_ids, moderator_user_ids, ta_user_ids): + """ + Create and return a function that determines user labels based on role. + + Args: + course_staff_user_ids: List of user IDs for course staff + moderator_user_ids: List of user IDs for moderators + ta_user_ids: List of user IDs for TAs + + Returns: + A function that takes a user_id and returns the appropriate label or None + """ + + def get_user_label(user_id): + """Get role label for a user ID.""" + try: + user_id_int = int(user_id) + if user_id_int in course_staff_user_ids: + return "Staff" + elif user_id_int in moderator_user_ids: + return "Moderator" + elif user_id_int in ta_user_ids: + return "Community TA" + except (ValueError, TypeError): + # If user_id has any issues, there's no label to return + pass + return None + + return get_user_label + + +def _process_deleted_thread(thread_data, get_user_label_fn, usernames_set): + """ + Process a single deleted thread into the standardized content item format. + + Args: + thread_data: Raw thread data from forum API + get_user_label_fn: Function to get user labels by user ID + usernames_set: Set to collect usernames for profile image fetch (modified in-place) + + Returns: + dict: Formatted content item for the thread + """ + author_username = thread_data.get("author_username", "") + deleted_by_id = thread_data.get("deleted_by") + deleted_by_username = None + + # Get deleted_by username + if deleted_by_id: + try: + deleted_user = User.objects.get(id=int(deleted_by_id)) + deleted_by_username = deleted_user.username + usernames_set.add(deleted_by_username) + except (User.DoesNotExist, ValueError): + # If user not found or invalid ID, skip setting deleted fields + pass + + if author_username: + usernames_set.add(author_username) + + # Strip HTML tags from preview + body_text = thread_data.get("body", "") + preview_text = strip_tags(body_text)[:100] if body_text else "" + + thread_id = thread_data.get("_id", thread_data.get("id")) + return { + "id": str(thread_id) + "-thread", + "type": "thread", + "title": thread_data.get("title", ""), + "body": body_text, + "preview_body": preview_text, + "course_id": thread_data.get("course_id", ""), + "author": author_username, + "author_id": thread_data.get("author_id", ""), + "author_label": get_user_label_fn(thread_data.get("author_id")), + "commentable_id": thread_data.get("commentable_id", ""), + "created_at": thread_data.get("created_at"), + "updated_at": thread_data.get("updated_at"), + "is_deleted": True, + "deleted_at": thread_data.get("deleted_at"), + "deleted_by": deleted_by_username, + "deleted_by_label": get_user_label_fn(deleted_by_id) if deleted_by_id else None, + "thread_type": thread_data.get("thread_type", "discussion"), + "anonymous": thread_data.get("anonymous", False), + "anonymous_to_peers": thread_data.get("anonymous_to_peers", False), + "vote_count": thread_data.get("vote_count", 0), + "comment_count": thread_data.get("comment_count", 0), + } + + +def _process_deleted_comment(comment_data, get_user_label_fn, usernames_set): + """ + Process a single deleted comment into the standardized content item format. + + Args: + comment_data: Raw comment data from forum API + get_user_label_fn: Function to get user labels by user ID + usernames_set: Set to collect usernames for profile image fetch (modified in-place) + + Returns: + dict: Formatted content item for the comment + """ + author_username = comment_data.get("author_username", "") + deleted_by_id = comment_data.get("deleted_by") + deleted_by_username = None + + # Get deleted_by username + if deleted_by_id: + try: + deleted_user = User.objects.get(id=int(deleted_by_id)) + deleted_by_username = deleted_user.username + usernames_set.add(deleted_by_username) + except (User.DoesNotExist, ValueError): + # If user not found or invalid ID, skip setting deleted fields + pass + + if author_username: + usernames_set.add(author_username) + + # Determine if this is a response (depth=0) or comment (depth>0) + depth = comment_data.get("depth", 0) + comment_type = "response" if depth == 0 else "comment" + + # Get parent thread title for context + thread_id = comment_data.get("comment_thread_id", "") + thread_title = "" + if thread_id: + try: + parent_thread = Thread(id=thread_id).retrieve() + thread_title = parent_thread.get("title", "") + except Exception: # pylint: disable=broad-exception-caught + pass + + # Strip HTML tags from preview + body_text = comment_data.get("body", "") + preview_text = strip_tags(body_text)[:100] if body_text else "" + + comment_id = comment_data.get("_id", comment_data.get("id")) + return { + "id": str(comment_id) + "-comment", + "type": comment_type, + "body": body_text, + "preview_body": preview_text, + "title": thread_title, # Use parent thread title for comments/responses + "course_id": comment_data.get("course_id", ""), + "author": author_username, + "author_id": comment_data.get("author_id", ""), + "author_label": get_user_label_fn(comment_data.get("author_id")), + "comment_thread_id": str(thread_id), + "thread_title": thread_title, + "parent_id": ( + str(comment_data.get("parent_id", "")) + if comment_data.get("parent_id") + else None + ), + "created_at": comment_data.get("created_at"), + "updated_at": comment_data.get("updated_at"), + "is_deleted": True, + "deleted_at": comment_data.get("deleted_at"), + "deleted_by": deleted_by_username, + "deleted_by_label": get_user_label_fn(deleted_by_id) if deleted_by_id else None, + "depth": depth, + "anonymous": comment_data.get("anonymous", False), + "anonymous_to_peers": comment_data.get("anonymous_to_peers", False), + "endorsed": comment_data.get("endorsed", False), + "vote_count": comment_data.get("vote_count", 0), + } + + +def _add_user_profiles_to_content(deleted_content, usernames_set, request): + """ + Fetch user profile images and add them to each content item. + + Args: + deleted_content: List of content items (modified in-place) + usernames_set: Set of usernames to fetch profile images for + request: Django request object for getting profile images + """ + # Add profile images for all users + username_profile_dict = _get_user_profile_dict( + request, usernames=",".join(usernames_set) + ) + + # Add users dict with profile images to each item + for item in deleted_content: + users_dict = {} + + # Add author profile + author_username = item.get("author") + if author_username and author_username in username_profile_dict: + users_dict[author_username] = _user_profile( + username_profile_dict[author_username] + ) + + # Add deleted_by profile + deleted_by_username = item.get("deleted_by") + if deleted_by_username and deleted_by_username in username_profile_dict: + users_dict[deleted_by_username] = _user_profile( + username_profile_dict[deleted_by_username] + ) + + item["users"] = users_dict + + +def get_deleted_content_for_course( + request, course_id, content_type=None, page=1, per_page=20, author_id=None +): + """ + Retrieve all deleted content (threads, comments) for a course. + + Args: + request: The django request object for getting user profile images + course_id (str): Course identifier + content_type (str, optional): Filter by 'thread' or 'comment'. If None, returns all types. + page (int): Page number for pagination (1-based) + per_page (int): Number of items per page + author_id (str, optional): Filter by author ID + + Returns: + dict: Paginated results with deleted content including author labels and profile images + """ + + import math + + from lms.djangoapps.discussion.rest_api.utils import ( + get_course_staff_users_list, + get_course_ta_users_list, + get_moderator_users_list, + ) + + try: + # Get course and user role information for labels + course_key = CourseKey.from_string(course_id) + course = _get_course(course_key, request.user) + + course_staff_user_ids = get_course_staff_users_list(course.id) + moderator_user_ids = get_moderator_users_list(course.id) + ta_user_ids = get_course_ta_users_list(course.id) + + # Get user label function + get_user_label = _get_user_label_function( + course_staff_user_ids, moderator_user_ids, ta_user_ids + ) + + # Build query parameters for forum API + query_params = { + "course_id": course_id, + "is_deleted": True, # Only get deleted content + "page": page, + "per_page": per_page, + } + + if author_id: + query_params["author_id"] = author_id + + deleted_content = [] + total_count = 0 + usernames_set = set() # Track all usernames for profile image fetch + + # Get deleted threads + if content_type is None or content_type == "thread": + try: + deleted_threads = forum_api.get_deleted_threads_for_course( + course_id=course_id, + page=page if content_type == "thread" else 1, + per_page=per_page if content_type == "thread" else 1000, + author_id=author_id, + ) + for thread_data in deleted_threads.get("threads", []): + content_item = _process_deleted_thread( + thread_data, get_user_label, usernames_set + ) + deleted_content.append(content_item) + + if content_type == "thread": + total_count = deleted_threads.get( + "total_count", len(deleted_content) + ) + except Exception as e: # pylint: disable=broad-exception-caught + log.warning( + "Failed to get deleted threads for course %s: %s", course_id, e + ) + + # Get deleted comments + if content_type is None or content_type == "comment": + try: + deleted_comments = forum_api.get_deleted_comments_for_course( + course_id=course_id, + page=page if content_type == "comment" else 1, + per_page=per_page if content_type == "comment" else 1000, + author_id=author_id, + ) + for comment_data in deleted_comments.get("comments", []): + content_item = _process_deleted_comment( + comment_data, get_user_label, usernames_set + ) + deleted_content.append(content_item) + + if content_type == "comment": + total_count = deleted_comments.get( + "total_count", len(deleted_content) + ) + except Exception as e: # pylint: disable=broad-exception-caught + log.warning( + "Failed to get deleted comments for course %s: %s", course_id, e + ) + + # If getting all content types, handle pagination differently + if content_type is None: + total_count = len(deleted_content) + # Sort by deletion date (most recent first) + deleted_content.sort(key=lambda x: x.get("deleted_at", ""), reverse=True) + + # Apply pagination to combined results + start_idx = (page - 1) * per_page + end_idx = start_idx + per_page + deleted_content = deleted_content[start_idx:end_idx] + + # Add profile images for all users + _add_user_profiles_to_content(deleted_content, usernames_set, request) + + # Calculate pagination info + num_pages = math.ceil(total_count / per_page) if total_count > 0 else 1 + + return { + "results": deleted_content, + "pagination": { + "next": None, # Can be computed if needed + "previous": None, # Can be computed if needed + "count": total_count, + "num_pages": num_pages, + }, + } + + except Exception as e: + log.exception("Error getting deleted content for course %s: %s", course_id, e) + raise diff --git a/lms/djangoapps/discussion/rest_api/forms.py b/lms/djangoapps/discussion/rest_api/forms.py index 8cc7127645b2..f37543723792 100644 --- a/lms/djangoapps/discussion/rest_api/forms.py +++ b/lms/djangoapps/discussion/rest_api/forms.py @@ -1,6 +1,7 @@ """ Discussion API forms """ + import urllib.parse from django.core.exceptions import ValidationError @@ -22,13 +23,15 @@ class UserOrdering(TextChoices): - BY_ACTIVITY = 'activity' - BY_FLAGS = 'flagged' - BY_RECENT_ACTIVITY = 'recency' + BY_ACTIVITY = "activity" + BY_FLAGS = "flagged" + BY_RECENT_ACTIVITY = "recency" + BY_DELETED = "deleted" class _PaginationForm(Form): """A form that includes pagination fields""" + page = IntegerField(required=False, min_value=1) page_size = IntegerField(required=False, min_value=1) @@ -45,6 +48,7 @@ class ThreadListGetForm(_PaginationForm): """ A form to validate query parameters in the thread list retrieval endpoint """ + EXCLUSIVE_PARAMS = ["topic_id", "text_search", "following"] course_id = CharField() @@ -58,17 +62,22 @@ class ThreadListGetForm(_PaginationForm): ) count_flagged = ExtendedNullBooleanField(required=False) flagged = ExtendedNullBooleanField(required=False) + show_deleted = ExtendedNullBooleanField(required=False) view = ChoiceField( - choices=[(choice, choice) for choice in ["unread", "unanswered", "unresponded"]], + choices=[ + (choice, choice) for choice in ["unread", "unanswered", "unresponded"] + ], required=False, ) order_by = ChoiceField( - choices=[(choice, choice) for choice in ["last_activity_at", "comment_count", "vote_count"]], - required=False + choices=[ + (choice, choice) + for choice in ["last_activity_at", "comment_count", "vote_count"] + ], + required=False, ) order_direction = ChoiceField( - choices=[(choice, choice) for choice in ["desc"]], - required=False + choices=[(choice, choice) for choice in ["desc"]], required=False ) requested_fields = MultiValueField(required=False) @@ -85,14 +94,16 @@ def clean_course_id(self): value = self.cleaned_data["course_id"] try: return CourseLocator.from_string(value) - except InvalidKeyError: - raise ValidationError(f"'{value}' is not a valid course id") # lint-amnesty, pylint: disable=raise-missing-from + except InvalidKeyError as e: + raise ValidationError(f"'{value}' is not a valid course id") from e def clean_following(self): """Validate following""" value = self.cleaned_data["following"] if value is False: # lint-amnesty, pylint: disable=no-else-raise - raise ValidationError("The value of the 'following' parameter must be true.") + raise ValidationError( + "The value of the 'following' parameter must be true." + ) else: return value @@ -115,6 +126,7 @@ class ThreadActionsForm(Form): A form to handle fields in thread creation/update that require separate interactions with the comments service. """ + following = BooleanField(required=False) voted = BooleanField(required=False) abuse_flagged = BooleanField(required=False) @@ -126,17 +138,20 @@ class CommentListGetForm(_PaginationForm): """ A form to validate query parameters in the comment list retrieval endpoint """ + thread_id = CharField() flagged = BooleanField(required=False) endorsed = ExtendedNullBooleanField(required=False) requested_fields = MultiValueField(required=False) merge_question_type_responses = BooleanField(required=False) + show_deleted = ExtendedNullBooleanField(required=False) class UserCommentListGetForm(_PaginationForm): """ A form to validate query parameters in the comment list retrieval endpoint """ + course_id = CharField() flagged = BooleanField(required=False) requested_fields = MultiValueField(required=False) @@ -146,8 +161,8 @@ def clean_course_id(self): value = self.cleaned_data["course_id"] try: return CourseLocator.from_string(value) - except InvalidKeyError: - raise ValidationError(f"'{value}' is not a valid course id") # lint-amnesty, pylint: disable=raise-missing-from + except InvalidKeyError as e: + raise ValidationError(f"'{value}' is not a valid course id") from e class CommentActionsForm(Form): @@ -155,6 +170,7 @@ class CommentActionsForm(Form): A form to handle fields in comment creation/update that require separate interactions with the comments service. """ + voted = BooleanField(required=False) abuse_flagged = BooleanField(required=False) @@ -163,6 +179,7 @@ class CommentGetForm(_PaginationForm): """ A form to validate query parameters in the comment retrieval endpoint """ + requested_fields = MultiValueField(required=False) @@ -170,28 +187,34 @@ class CourseDiscussionSettingsForm(Form): """ A form to validate the fields in the course discussion settings requests. """ + course_id = CharField() def __init__(self, *args, **kwargs): - self.request_user = kwargs.pop('request_user') + self.request_user = kwargs.pop("request_user") super().__init__(*args, **kwargs) def clean_course_id(self): """Validate the 'course_id' value""" - course_id = self.cleaned_data['course_id'] + course_id = self.cleaned_data["course_id"] try: course_key = CourseKey.from_string(course_id) - self.cleaned_data['course'] = get_course_with_access(self.request_user, 'load', course_key) - self.cleaned_data['course_key'] = course_key + self.cleaned_data["course"] = get_course_with_access( + self.request_user, "load", course_key + ) + self.cleaned_data["course_key"] = course_key return course_id - except InvalidKeyError: - raise ValidationError(f"'{str(course_id)}' is not a valid course key") # lint-amnesty, pylint: disable=raise-missing-from + except InvalidKeyError as e: + raise ValidationError( + f"'{str(course_id)}' is not a valid course key" + ) from e class CourseDiscussionRolesForm(CourseDiscussionSettingsForm): """ A form to validate the fields in the course discussion roles requests. """ + ROLE_CHOICES = ( (FORUM_ROLE_MODERATOR, FORUM_ROLE_MODERATOR), (FORUM_ROLE_COMMUNITY_TA, FORUM_ROLE_MODERATOR), @@ -199,20 +222,20 @@ class CourseDiscussionRolesForm(CourseDiscussionSettingsForm): ) rolename = ChoiceField( choices=ROLE_CHOICES, - error_messages={"invalid_choice": "Role '%(value)s' does not exist"} + error_messages={"invalid_choice": "Role '%(value)s' does not exist"}, ) def clean_rolename(self): """Validate the 'rolename' value.""" - rolename = urllib.parse.unquote(self.cleaned_data.get('rolename')) - course_id = self.cleaned_data.get('course_key') + rolename = urllib.parse.unquote(self.cleaned_data.get("rolename")) + course_id = self.cleaned_data.get("course_key") if course_id and rolename: try: role = Role.objects.get(name=rolename, course_id=course_id) except Role.DoesNotExist as err: raise ValidationError(f"Role '{rolename}' does not exist") from err - self.cleaned_data['role'] = role + self.cleaned_data["role"] = role return rolename @@ -220,15 +243,17 @@ class TopicListGetForm(Form): """ Form for the topics API get query parameters. """ + topic_id = CharField(required=False) order_by = ChoiceField(choices=TopicOrdering.choices, required=False) def clean_topic_id(self): topic_ids = self.cleaned_data.get("topic_id", None) - return set(topic_ids.strip(',').split(',')) if topic_ids else None + return set(topic_ids.strip(",").split(",")) if topic_ids else None class CourseActivityStatsForm(_PaginationForm): """Form for validating course activity stats API query parameters""" + order_by = ChoiceField(choices=UserOrdering.choices, required=False) username = CharField(required=False) diff --git a/lms/djangoapps/discussion/rest_api/serializers.py b/lms/djangoapps/discussion/rest_api/serializers.py index 8a7ab16e0903..902a433dac3b 100644 --- a/lms/djangoapps/discussion/rest_api/serializers.py +++ b/lms/djangoapps/discussion/rest_api/serializers.py @@ -1,13 +1,13 @@ """ Discussion API serializers """ + import html import re - -from bs4 import BeautifulSoup from typing import Dict from urllib.parse import urlencode, urlunparse +from bs4 import BeautifulSoup from django.conf import settings from django.contrib.auth import get_user_model from django.core.exceptions import ObjectDoesNotExist, ValidationError @@ -18,8 +18,12 @@ from common.djangoapps.student.models import get_user_by_username_or_email from common.djangoapps.student.roles import GlobalStaff -from lms.djangoapps.discussion.django_comment_client.base.views import track_thread_lock_unlock_event, \ - track_thread_edited_event, track_comment_edited_event, track_forum_response_mark_event +from lms.djangoapps.discussion.django_comment_client.base.views import ( + track_comment_edited_event, + track_forum_response_mark_event, + track_thread_edited_event, + track_thread_lock_unlock_event, +) from lms.djangoapps.discussion.django_comment_client.utils import ( course_discussion_division_enabled, get_group_id_for_user, @@ -35,17 +39,23 @@ from lms.djangoapps.discussion.rest_api.render import render_body from lms.djangoapps.discussion.rest_api.utils import ( get_course_staff_users_list, - get_moderator_users_list, get_course_ta_users_list, + get_moderator_users_list, get_user_learner_status, ) from openedx.core.djangoapps.discussions.models import DiscussionTopicLink from openedx.core.djangoapps.discussions.utils import get_group_names_by_id from openedx.core.djangoapps.django_comment_common.comment_client.comment import Comment from openedx.core.djangoapps.django_comment_common.comment_client.thread import Thread -from openedx.core.djangoapps.django_comment_common.comment_client.user import User as CommentClientUser -from openedx.core.djangoapps.django_comment_common.comment_client.utils import CommentClientRequestError -from openedx.core.djangoapps.django_comment_common.models import CourseDiscussionSettings +from openedx.core.djangoapps.django_comment_common.comment_client.user import ( + User as CommentClientUser, +) +from openedx.core.djangoapps.django_comment_common.comment_client.utils import ( + CommentClientRequestError, +) +from openedx.core.djangoapps.django_comment_common.models import ( + CourseDiscussionSettings, +) from openedx.core.djangoapps.user_api.accounts.api import get_profile_images from openedx.core.lib.api.serializers import CourseKeyField @@ -59,6 +69,7 @@ class TopicOrdering(TextChoices): """ Enum for the available options for ordering topics. """ + COURSE_STRUCTURE = "course_structure", "Course Structure" ACTIVITY = "activity", "Activity" NAME = "name", "Name" @@ -73,16 +84,24 @@ def get_context(course, request, thread=None): moderator_user_ids = get_moderator_users_list(course.id) ta_user_ids = get_course_ta_users_list(course.id) requester = request.user - cc_requester = CommentClientUser.from_django_user(requester).retrieve(course_id=course.id) + cc_requester = CommentClientUser.from_django_user(requester).retrieve( + course_id=course.id + ) cc_requester["course_id"] = course.id course_discussion_settings = CourseDiscussionSettings.get(course.id) is_global_staff = GlobalStaff().has_user(requester) - has_moderation_privilege = requester.id in moderator_user_ids or requester.id in ta_user_ids or is_global_staff + has_moderation_privilege = ( + requester.id in moderator_user_ids + or requester.id in ta_user_ids + or is_global_staff + ) return { "course": course, "request": request, "thread": thread, - "discussion_division_enabled": course_discussion_division_enabled(course_discussion_settings), + "discussion_division_enabled": course_discussion_division_enabled( + course_discussion_settings + ), "group_ids_to_names": get_group_names_by_id(course_discussion_settings), "moderator_user_ids": moderator_user_ids, "course_staff_user_ids": course_staff_user_ids, @@ -137,8 +156,8 @@ def _validate_privileged_access(context: Dict) -> bool: Returns: bool: Course exists and the user has privileged access. """ - course = context.get('course', None) - is_requester_privileged = context.get('has_moderation_privilege') + course = context.get("course", None) + is_requester_privileged = context.get("has_moderation_privilege") return course and is_requester_privileged @@ -158,7 +177,7 @@ def filter_spam_urls_from_html(html_string): patterns.append(re.compile(rf"(https?://)?{domain_pattern}", re.IGNORECASE)) for a_tag in soup.find_all("a", href=True): - href = a_tag.get('href') + href = a_tag.get("href") if href: if any(p.search(href) for p in patterns): a_tag.replace_with(a_tag.get_text(strip=True)) @@ -167,7 +186,7 @@ def filter_spam_urls_from_html(html_string): for text_node in soup.find_all(string=True): new_text = text_node for p in patterns: - new_text = p.sub('', new_text) + new_text = p.sub("", new_text) if new_text != text_node: text_node.replace_with(new_text.strip()) is_spam = True @@ -196,8 +215,14 @@ class _ContentSerializer(serializers.Serializer): anonymous = serializers.BooleanField(default=False) anonymous_to_peers = serializers.BooleanField(default=False) last_edit = serializers.SerializerMethodField(required=False) - edit_reason_code = serializers.CharField(required=False, validators=[validate_edit_reason_code]) + edit_reason_code = serializers.CharField( + required=False, validators=[validate_edit_reason_code] + ) edit_by_label = serializers.SerializerMethodField(required=False) + is_deleted = serializers.SerializerMethodField(read_only=True) + deleted_at = serializers.SerializerMethodField(read_only=True) + deleted_by = serializers.SerializerMethodField(read_only=True) + deleted_by_label = serializers.SerializerMethodField(read_only=True) non_updatable_fields = set() @@ -219,7 +244,10 @@ def _is_user_privileged(self, user_id): Returns a boolean indicating whether the given user_id identifies a privileged user. """ - return user_id in self.context["moderator_user_ids"] or user_id in self.context["ta_user_ids"] + return ( + user_id in self.context["moderator_user_ids"] + or user_id in self.context["ta_user_ids"] + ) def _is_anonymous(self, obj): """ @@ -227,13 +255,13 @@ def _is_anonymous(self, obj): the requester. """ user_id = self.context["request"].user.id - is_user_staff = user_id in self.context["moderator_user_ids"] or user_id in self.context["ta_user_ids"] - - return ( - obj["anonymous"] or - obj["anonymous_to_peers"] and not is_user_staff + is_user_staff = ( + user_id in self.context["moderator_user_ids"] + or user_id in self.context["ta_user_ids"] ) + return obj["anonymous"] or obj["anonymous_to_peers"] and not is_user_staff + def get_author(self, obj): """ Returns the author's username, or None if the content is anonymous. @@ -250,10 +278,9 @@ def _get_user_label(self, user_id): is_ta = user_id in self.context["ta_user_ids"] return ( - "Staff" if is_staff else - "Moderator" if is_moderator else - "Community TA" if is_ta else - None + "Staff" + if is_staff + else "Moderator" if is_moderator else "Community TA" if is_ta else None ) def _get_user_label_from_username(self, username): @@ -303,7 +330,9 @@ def get_rendered_body(self, obj): """ if self._rendered_body is None: self._rendered_body = render_body(obj["body"]) - self._rendered_body, is_spam = filter_spam_urls_from_html(self._rendered_body) + self._rendered_body, is_spam = filter_spam_urls_from_html( + self._rendered_body + ) if is_spam and settings.CONTENT_FOR_SPAM_POSTS: self._rendered_body = settings.CONTENT_FOR_SPAM_POSTS return self._rendered_body @@ -315,8 +344,9 @@ def get_abuse_flagged(self, obj): """ total_abuse_flaggers = len(obj.get("abuse_flaggers", [])) return ( - self.context["has_moderation_privilege"] and total_abuse_flaggers > 0 or - self.context["cc_requester"]["id"] in obj.get("abuse_flaggers", []) + self.context["has_moderation_privilege"] + and total_abuse_flaggers > 0 + or self.context["cc_requester"]["id"] in obj.get("abuse_flaggers", []) ) def get_voted(self, obj): @@ -349,7 +379,7 @@ def get_last_edit(self, obj): Returns information about the last edit for this content for privileged users. """ - is_user_author = str(obj['user_id']) == str(self.context['request'].user.id) + is_user_author = str(obj["user_id"]) == str(self.context["request"].user.id) if not (_validate_privileged_access(self.context) or is_user_author): return None edit_history = obj.get("edit_history") @@ -365,12 +395,57 @@ def get_edit_by_label(self, obj): """ Returns the role label for the last edit user. """ - is_user_author = str(obj['user_id']) == str(self.context['request'].user.id) + is_user_author = str(obj["user_id"]) == str(self.context["request"].user.id) is_user_privileged = _validate_privileged_access(self.context) edit_history = obj.get("edit_history") if (is_user_author or is_user_privileged) and edit_history: last_edit = edit_history[-1] - return self._get_user_label_from_username(last_edit.get('editor_username')) + return self._get_user_label_from_username(last_edit.get("editor_username")) + + def get_is_deleted(self, obj): + """ + Returns the is_deleted status for privileged users only. + """ + if not _validate_privileged_access(self.context): + return None + return obj.get("is_deleted", False) + + def get_deleted_at(self, obj): + """ + Returns the deletion timestamp for privileged users only. + """ + if not _validate_privileged_access(self.context): + return None + return obj.get("deleted_at") + + def get_deleted_by(self, obj): + """ + Returns the username of the user who deleted this content for privileged users only. + """ + if not _validate_privileged_access(self.context): + return None + deleted_by_id = obj.get("deleted_by") + if deleted_by_id: + try: + user = User.objects.get(id=int(deleted_by_id)) + return user.username + except (User.DoesNotExist, ValueError): + return None + return None + + def get_deleted_by_label(self, obj): + """ + Returns the role label for the user who deleted this content for privileged users only. + """ + if not _validate_privileged_access(self.context): + return None + deleted_by_id = obj.get("deleted_by") + if deleted_by_id: + try: + return self._get_user_label(int(deleted_by_id)) + except (ValueError, TypeError): + return None + return None class ThreadSerializer(_ContentSerializer): @@ -381,13 +456,15 @@ class ThreadSerializer(_ContentSerializer): not had retrieve() called, because of the interaction between DRF's attempts at introspection and Thread's __getattr__. """ + course_id = serializers.CharField() - topic_id = serializers.CharField(source="commentable_id", validators=[validate_not_blank]) + topic_id = serializers.CharField( + source="commentable_id", validators=[validate_not_blank] + ) group_id = serializers.IntegerField(required=False, allow_null=True) group_name = serializers.SerializerMethodField() type = serializers.ChoiceField( - source="thread_type", - choices=[(val, val) for val in ["discussion", "question"]] + source="thread_type", choices=[(val, val) for val in ["discussion", "question"]] ) preview_body = serializers.SerializerMethodField() abuse_flagged_count = serializers.SerializerMethodField(required=False) @@ -402,8 +479,12 @@ class ThreadSerializer(_ContentSerializer): non_endorsed_comment_list_url = serializers.SerializerMethodField() read = serializers.BooleanField(required=False) has_endorsed = serializers.BooleanField(source="endorsed", read_only=True) - response_count = serializers.IntegerField(source="resp_total", read_only=True, required=False) - close_reason_code = serializers.CharField(required=False, validators=[validate_close_reason_code]) + response_count = serializers.IntegerField( + source="resp_total", read_only=True, required=False + ) + close_reason_code = serializers.CharField( + required=False, validators=[validate_close_reason_code] + ) close_reason = serializers.SerializerMethodField() closed_by = serializers.SerializerMethodField() closed_by_label = serializers.SerializerMethodField(required=False) @@ -449,9 +530,8 @@ def get_comment_list_url(self, obj, endorsed=None): Returns the URL to retrieve the thread's comments, optionally including the endorsed query parameter. """ - if ( - (obj["thread_type"] == "question" and endorsed is None) or - (obj["thread_type"] == "discussion" and endorsed is not None) + if (obj["thread_type"] == "question" and endorsed is None) or ( + obj["thread_type"] == "discussion" and endorsed is not None ): return None path = reverse("comment-list") @@ -495,13 +575,17 @@ def get_preview_body(self, obj): """ Returns a cleaned version of the thread's body to display in a preview capacity. """ - return strip_tags(self.get_rendered_body(obj)).replace('\n', ' ').replace(' ', ' ') + return ( + strip_tags(self.get_rendered_body(obj)) + .replace("\n", " ") + .replace(" ", " ") + ) def get_close_reason(self, obj): """ Returns the reason for which the thread was closed. """ - is_user_author = str(obj['user_id']) == str(self.context['request'].user.id) + is_user_author = str(obj["user_id"]) == str(self.context["request"].user.id) if not (_validate_privileged_access(self.context) or is_user_author): return None reason_code = obj.get("close_reason_code") @@ -512,7 +596,7 @@ def get_closed_by(self, obj): Returns the username of the moderator who closed this thread, only to other privileged users and author. """ - is_user_author = str(obj['user_id']) == str(self.context['request'].user.id) + is_user_author = str(obj["user_id"]) == str(self.context["request"].user.id) if _validate_privileged_access(self.context) or is_user_author: return obj.get("closed_by") @@ -520,7 +604,7 @@ def get_closed_by_label(self, obj): """ Returns the role label for the user who closed the post. """ - is_user_author = str(obj['user_id']) == str(self.context['request'].user.id) + is_user_author = str(obj["user_id"]) == str(self.context["request"].user.id) if is_user_author or _validate_privileged_access(self.context): return self._get_user_label_from_username(obj.get("closed_by")) @@ -535,18 +619,31 @@ def update(self, instance, validated_data): requesting_user_id = self.context["cc_requester"]["id"] if key == "closed" and val: instance["closing_user_id"] = requesting_user_id - track_thread_lock_unlock_event(self.context['request'], self.context['course'], - instance, validated_data.get('close_reason_code')) + track_thread_lock_unlock_event( + self.context["request"], + self.context["course"], + instance, + validated_data.get("close_reason_code"), + ) if key == "closed" and not val: instance["closing_user_id"] = requesting_user_id - track_thread_lock_unlock_event(self.context['request'], self.context['course'], - instance, validated_data.get('close_reason_code'), locked=False) + track_thread_lock_unlock_event( + self.context["request"], + self.context["course"], + instance, + validated_data.get("close_reason_code"), + locked=False, + ) if key == "body" and val: instance["editing_user_id"] = requesting_user_id - track_thread_edited_event(self.context['request'], self.context['course'], - instance, validated_data.get('edit_reason_code')) + track_thread_edited_event( + self.context["request"], + self.context["course"], + instance, + validated_data.get("edit_reason_code"), + ) instance.save() return instance @@ -559,6 +656,7 @@ class CommentSerializer(_ContentSerializer): not had retrieve() called, because of the interaction between DRF's attempts at introspection and Comment's __getattr__. """ + thread_id = serializers.CharField() parent_id = serializers.CharField(required=False, allow_null=True) endorsed = serializers.BooleanField(required=False) @@ -573,7 +671,7 @@ class CommentSerializer(_ContentSerializer): non_updatable_fields = NON_UPDATABLE_COMMENT_FIELDS def __init__(self, *args, **kwargs): - remove_fields = kwargs.pop('remove_fields', None) + remove_fields = kwargs.pop("remove_fields", None) super().__init__(*args, **kwargs) if remove_fields: @@ -595,8 +693,8 @@ def get_endorsed_by(self, obj): # Avoid revealing the identity of an anonymous non-staff question # author who has endorsed a comment in the thread if not ( - self._is_anonymous(self.context["thread"]) and - not self._is_user_privileged(endorser_id) + self._is_anonymous(self.context["thread"]) + and not self._is_user_privileged(endorser_id) ): return User.objects.get(id=endorser_id).username return None @@ -638,7 +736,7 @@ def to_representation(self, data): # Django Rest Framework v3 no longer includes None values # in the representation. To maintain the previous behavior, # we do this manually instead. - if 'parent_id' not in data: + if "parent_id" not in data: data["parent_id"] = None return data @@ -680,7 +778,7 @@ def create(self, validated_data): comment = Comment( course_id=self.context["thread"]["course_id"], user_id=self.context["cc_requester"]["id"], - **validated_data + **validated_data, ) comment.save() return comment @@ -693,12 +791,18 @@ def update(self, instance, validated_data): # endorsement_user_id on update requesting_user_id = self.context["cc_requester"]["id"] if key == "endorsed": - track_forum_response_mark_event(self.context['request'], self.context['course'], instance, val) + track_forum_response_mark_event( + self.context["request"], self.context["course"], instance, val + ) instance["endorsement_user_id"] = requesting_user_id if key == "body" and val: instance["editing_user_id"] = requesting_user_id - track_comment_edited_event(self.context['request'], self.context['course'], - instance, validated_data.get('edit_reason_code')) + track_comment_edited_event( + self.context["request"], + self.context["course"], + instance, + validated_data.get("edit_reason_code"), + ) instance.save() return instance @@ -708,6 +812,7 @@ class DiscussionTopicSerializer(serializers.Serializer): """ Serializer for DiscussionTopic """ + id = serializers.CharField(read_only=True) # pylint: disable=invalid-name name = serializers.CharField(read_only=True) thread_list_url = serializers.CharField(read_only=True) @@ -737,10 +842,11 @@ class DiscussionTopicSerializerV2(serializers.Serializer): """ Serializer for new style topics. """ + id = serializers.CharField( # pylint: disable=invalid-name read_only=True, source="external_id", - help_text="Provider-specific unique id for the topic" + help_text="Provider-specific unique id for the topic", ) usage_key = serializers.CharField( read_only=True, @@ -764,10 +870,13 @@ def get_thread_counts(self, obj: DiscussionTopicLink) -> Dict[str, int]: """ Get thread counts from provided context """ - return self.context['thread_counts'].get(obj.external_id, { - "discussion": 0, - "question": 0, - }) + return self.context["thread_counts"].get( + obj.external_id, + { + "discussion": 0, + "question": 0, + }, + ) class DiscussionRolesSerializer(serializers.Serializer): @@ -775,10 +884,7 @@ class DiscussionRolesSerializer(serializers.Serializer): Serializer for course discussion roles. """ - ACTION_CHOICES = ( - ('allow', 'allow'), - ('revoke', 'revoke') - ) + ACTION_CHOICES = (("allow", "allow"), ("revoke", "revoke")) action = serializers.ChoiceField(ACTION_CHOICES) user_id = serializers.CharField() @@ -799,14 +905,16 @@ def validate_user_id(self, user_id): self.user = get_user_by_username_or_email(user_id) return user_id except User.DoesNotExist as err: - raise ValidationError(f"'{user_id}' is not a valid student identifier") from err + raise ValidationError( + f"'{user_id}' is not a valid student identifier" + ) from err def validate(self, attrs): """Validate the data at an object level.""" # Store the user object to avoid fetching it again. - if hasattr(self, 'user'): - attrs['user'] = self.user + if hasattr(self, "user"): + attrs["user"] = self.user return attrs def create(self, validated_data): @@ -824,6 +932,7 @@ class DiscussionRolesMemberSerializer(serializers.Serializer): """ Serializer for course discussion roles member data. """ + username = serializers.CharField() email = serializers.EmailField() first_name = serializers.CharField() @@ -832,7 +941,7 @@ class DiscussionRolesMemberSerializer(serializers.Serializer): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) - self.course_discussion_settings = self.context['course_discussion_settings'] + self.course_discussion_settings = self.context["course_discussion_settings"] def get_group_name(self, instance): """Return the group name of the user.""" @@ -855,6 +964,7 @@ class DiscussionRolesListSerializer(serializers.Serializer): """ Serializer for course discussion roles member list. """ + course_id = serializers.CharField() results = serializers.SerializerMethodField() division_scheme = serializers.SerializerMethodField() @@ -862,15 +972,17 @@ class DiscussionRolesListSerializer(serializers.Serializer): def get_results(self, obj): """Return the nested serializer data representing a list of member users.""" context = { - 'course_id': obj['course_id'], - 'course_discussion_settings': self.context['course_discussion_settings'] + "course_id": obj["course_id"], + "course_discussion_settings": self.context["course_discussion_settings"], } - serializer = DiscussionRolesMemberSerializer(obj['users'], context=context, many=True) + serializer = DiscussionRolesMemberSerializer( + obj["users"], context=context, many=True + ) return serializer.data def get_division_scheme(self, obj): # pylint: disable=unused-argument """Return the division scheme for the course.""" - return self.context['course_discussion_settings'].division_scheme + return self.context["course_discussion_settings"].division_scheme def create(self, validated_data): """ @@ -887,9 +999,13 @@ class UserStatsSerializer(serializers.Serializer): """ Serializer for course user stats. """ + threads = serializers.IntegerField() replies = serializers.IntegerField() responses = serializers.IntegerField() + deleted_threads = serializers.IntegerField(required=False, default=0) + deleted_replies = serializers.IntegerField(required=False, default=0) + deleted_responses = serializers.IntegerField(required=False, default=0) active_flags = serializers.IntegerField() inactive_flags = serializers.IntegerField() username = serializers.CharField() @@ -907,27 +1023,36 @@ class BlackoutDateSerializer(serializers.Serializer): """ Serializer for blackout dates. """ - start = serializers.DateTimeField(help_text="The ISO 8601 timestamp for the start of the blackout period") - end = serializers.DateTimeField(help_text="The ISO 8601 timestamp for the end of the blackout period") + + start = serializers.DateTimeField( + help_text="The ISO 8601 timestamp for the start of the blackout period" + ) + end = serializers.DateTimeField( + help_text="The ISO 8601 timestamp for the end of the blackout period" + ) class ReasonCodeSeralizer(serializers.Serializer): """ Serializer for reason codes. """ + code = serializers.CharField(help_text="A code for the an edit or close reason") - label = serializers.CharField(help_text="A user-friendly name text for the close or edit reason") + label = serializers.CharField( + help_text="A user-friendly name text for the close or edit reason" + ) class CourseMetadataSerailizer(serializers.Serializer): """ Serializer for course metadata. """ + id = CourseKeyField(help_text="The identifier of the course") blackouts = serializers.ListField( child=BlackoutDateSerializer(), help_text="A list of objects representing blackout periods " - "(during which discussions are read-only except for privileged users)." + "(during which discussions are read-only except for privileged users).", ) thread_list_url = serializers.URLField( help_text="The URL of the list of all threads in the course.", @@ -935,7 +1060,9 @@ class CourseMetadataSerailizer(serializers.Serializer): following_thread_list_url = serializers.URLField( help_text="thread_list_url with parameter following=True", ) - topics_url = serializers.URLField(help_text="The URL of the topic listing for the course.") + topics_url = serializers.URLField( + help_text="The URL of the topic listing for the course." + ) allow_anonymous = serializers.BooleanField( help_text="A boolean indicating whether anonymous posts are allowed or not.", ) diff --git a/lms/djangoapps/discussion/rest_api/tasks.py b/lms/djangoapps/discussion/rest_api/tasks.py index cd725a3513dc..5773fbbc83b0 100644 --- a/lms/djangoapps/discussion/rest_api/tasks.py +++ b/lms/djangoapps/discussion/rest_api/tasks.py @@ -1,32 +1,36 @@ """ Contain celery tasks """ + import logging from celery import shared_task from django.contrib.auth import get_user_model from edx_django_utils.monitoring import set_code_owner_attribute -from opaque_keys.edx.locator import CourseKey from eventtracking import tracker +from opaque_keys.edx.locator import CourseKey -from common.djangoapps.student.roles import CourseStaffRole, CourseInstructorRole +from common.djangoapps.student.roles import CourseInstructorRole, CourseStaffRole from common.djangoapps.track import segment from lms.djangoapps.courseware.courses import get_course_with_access from lms.djangoapps.discussion.django_comment_client.utils import get_user_role_names -from lms.djangoapps.discussion.rest_api.discussions_notifications import DiscussionNotificationSender +from lms.djangoapps.discussion.rest_api.discussions_notifications import ( + DiscussionNotificationSender, +) from lms.djangoapps.discussion.rest_api.utils import can_user_notify_all_learners from openedx.core.djangoapps.django_comment_common.comment_client import Comment from openedx.core.djangoapps.django_comment_common.comment_client.thread import Thread from openedx.core.djangoapps.notifications.config.waffle import ENABLE_NOTIFICATIONS - User = get_user_model() log = logging.getLogger(__name__) @shared_task @set_code_owner_attribute -def send_thread_created_notification(thread_id, course_key_str, user_id, notify_all_learners=False): +def send_thread_created_notification( + thread_id, course_key_str, user_id, notify_all_learners=False +): """ Send notification when a new thread is created """ @@ -40,17 +44,21 @@ def send_thread_created_notification(thread_id, course_key_str, user_id, notify_ is_course_staff = CourseStaffRole(course_key).has_user(user) is_course_admin = CourseInstructorRole(course_key).has_user(user) user_roles = get_user_role_names(user, course_key) - if not can_user_notify_all_learners(user_roles, is_course_staff, is_course_admin): + if not can_user_notify_all_learners( + user_roles, is_course_staff, is_course_admin + ): return - course = get_course_with_access(user, 'load', course_key, check_if_enrolled=True) + course = get_course_with_access(user, "load", course_key, check_if_enrolled=True) notification_sender = DiscussionNotificationSender(thread, course, user) notification_sender.send_new_thread_created_notification(notify_all_learners) @shared_task @set_code_owner_attribute -def send_response_notifications(thread_id, course_key_str, user_id, comment_id, parent_id=None): +def send_response_notifications( + thread_id, course_key_str, user_id, comment_id, parent_id=None +): """ Send notifications to users who are subscribed to the thread. """ @@ -59,8 +67,10 @@ def send_response_notifications(thread_id, course_key_str, user_id, comment_id, return thread = Thread(id=thread_id).retrieve() user = User.objects.get(id=user_id) - course = get_course_with_access(user, 'load', course_key, check_if_enrolled=True) - notification_sender = DiscussionNotificationSender(thread, course, user, parent_id, comment_id) + course = get_course_with_access(user, "load", course_key, check_if_enrolled=True) + notification_sender = DiscussionNotificationSender( + thread, course, user, parent_id, comment_id + ) notification_sender.send_new_comment_notification() notification_sender.send_new_response_notification() notification_sender.send_new_comment_on_response_notification() @@ -69,7 +79,9 @@ def send_response_notifications(thread_id, course_key_str, user_id, comment_id, @shared_task @set_code_owner_attribute -def send_response_endorsed_notifications(thread_id, response_id, course_key_str, endorsed_by): +def send_response_endorsed_notifications( + thread_id, response_id, course_key_str, endorsed_by +): """ Send notifications when a response is marked answered/ endorsed """ @@ -80,8 +92,10 @@ def send_response_endorsed_notifications(thread_id, response_id, course_key_str, response = Comment(id=response_id).retrieve() creator = User.objects.get(id=response.user_id) endorser = User.objects.get(id=endorsed_by) - course = get_course_with_access(creator, 'load', course_key, check_if_enrolled=True) - notification_sender = DiscussionNotificationSender(thread, course, creator, comment_id=response_id) + course = get_course_with_access(creator, "load", course_key, check_if_enrolled=True) + notification_sender = DiscussionNotificationSender( + thread, course, creator, comment_id=response_id + ) # skip sending notification to author of thread if they are the same as the author of the response if response.user_id != thread.user_id: # sends notification to author of thread @@ -99,15 +113,63 @@ def delete_course_post_for_user(user_id, username, course_ids, event_data=None): Deletes all posts for user in a course. """ event_data = event_data or {} - log.info(f"<> Deleting all posts for {username} in course {course_ids}") - threads_deleted = Thread.delete_user_threads(user_id, course_ids) - comments_deleted = Comment.delete_user_comments(user_id, course_ids) - log.info(f"<> Deleted {threads_deleted} posts and {comments_deleted} comments for {username} " - f"in course {course_ids}") - event_data.update({ - "number_of_posts_deleted": threads_deleted, - "number_of_comments_deleted": comments_deleted, - }) - event_name = 'edx.discussion.bulk_delete_user_posts' + log.info( + f"<> Deleting all posts for {username} in course {course_ids}" + ) + # Get triggered_by user_id from event_data for audit trail + deleted_by_user_id = event_data.get("triggered_by_user_id") if event_data else None + threads_deleted = Thread.delete_user_threads( + user_id, course_ids, deleted_by=deleted_by_user_id + ) + comments_deleted = Comment.delete_user_comments( + user_id, course_ids, deleted_by=deleted_by_user_id + ) + log.info( + f"<> Deleted {threads_deleted} posts and {comments_deleted} comments for {username} " + f"in course {course_ids}" + ) + event_data.update( + { + "number_of_posts_deleted": threads_deleted, + "number_of_comments_deleted": comments_deleted, + } + ) + event_name = "edx.discussion.bulk_delete_user_posts" + tracker.emit(event_name, event_data) + segment.track("None", event_name, event_data) + + +@shared_task +@set_code_owner_attribute +def restore_course_post_for_user(user_id, username, course_ids, event_data=None): + """ + Restores all soft-deleted posts for user in a course by setting is_deleted=False. + """ + event_data = event_data or {} + log.info( + "<> Restoring all posts for %s in course %s", username, course_ids + ) + # Get triggered_by user_id from event_data for audit trail + restored_by_user_id = event_data.get("triggered_by_user_id") if event_data else None + threads_restored = Thread.restore_user_deleted_threads( + user_id, course_ids, restored_by=restored_by_user_id + ) + comments_restored = Comment.restore_user_deleted_comments( + user_id, course_ids, restored_by=restored_by_user_id + ) + log.info( + "<> Restored %s posts and %s comments for %s in course %s", + threads_restored, + comments_restored, + username, + course_ids, + ) + event_data.update( + { + "number_of_posts_restored": threads_restored, + "number_of_comments_restored": comments_restored, + } + ) + event_name = "edx.discussion.bulk_restore_user_posts" tracker.emit(event_name, event_data) - segment.track('None', event_name, event_data) + segment.track("None", event_name, event_data) diff --git a/lms/djangoapps/discussion/rest_api/tests/test_api_v2.py b/lms/djangoapps/discussion/rest_api/tests/test_api_v2.py index 53c12454aec9..2fa761b46615 100644 --- a/lms/djangoapps/discussion/rest_api/tests/test_api_v2.py +++ b/lms/djangoapps/discussion/rest_api/tests/test_api_v2.py @@ -10,34 +10,20 @@ import random from datetime import datetime, timedelta from unittest import mock -from urllib.parse import parse_qs, urlencode, urlparse, urlunparse import ddt import httpretty import pytest -from django.test import override_settings from django.contrib.auth import get_user_model from django.core.exceptions import ValidationError from django.test.client import RequestFactory -from opaque_keys.edx.keys import CourseKey from opaque_keys.edx.locator import CourseLocator from pytz import UTC from rest_framework.exceptions import PermissionDenied -from xmodule.modulestore import ModuleStoreEnum -from xmodule.modulestore.django import modulestore -from xmodule.modulestore.tests.django_utils import ( - ModuleStoreTestCase, - SharedModuleStoreTestCase, -) -from xmodule.modulestore.tests.factories import CourseFactory, BlockFactory -from xmodule.partitions.partitions import Group, UserPartition - from common.djangoapps.student.tests.factories import ( AdminFactory, - BetaTesterFactory, CourseEnrollmentFactory, - StaffFactory, UserFactory, ) from common.djangoapps.util.testing import UrlResetMixin @@ -45,10 +31,6 @@ from lms.djangoapps.discussion.django_comment_client.tests.utils import ( ForumsEnableMixin, ) -from lms.djangoapps.discussion.tests.utils import ( - make_minimal_cs_comment, - make_minimal_cs_thread, -) from lms.djangoapps.discussion.rest_api import api from lms.djangoapps.discussion.rest_api.api import ( create_comment, @@ -56,12 +38,9 @@ delete_comment, delete_thread, get_comment_list, - get_course, - get_course_topics, get_course_topics_v2, get_thread, get_thread_list, - get_user_comments, update_comment, update_thread, ) @@ -73,18 +52,19 @@ ) from lms.djangoapps.discussion.rest_api.serializers import TopicOrdering from lms.djangoapps.discussion.rest_api.tests.utils import ( - CommentsServiceMockMixin, ForumMockUtilsMixin, make_paginated_api_response, - parsed_body, ) -from openedx.core.djangoapps.course_groups.models import CourseUserGroupPartitionGroup +from lms.djangoapps.discussion.tests.utils import ( + make_minimal_cs_comment, + make_minimal_cs_thread, +) from openedx.core.djangoapps.course_groups.tests.helpers import CohortFactory from openedx.core.djangoapps.discussions.models import ( DiscussionsConfiguration, DiscussionTopicLink, - Provider, PostingRestriction, + Provider, ) from openedx.core.djangoapps.discussions.tasks import ( update_discussions_settings_from_course_task, @@ -98,6 +78,13 @@ Role, ) from openedx.core.lib.exceptions import CourseNotFoundError, PageNotFoundError +from xmodule.modulestore import ModuleStoreEnum +from xmodule.modulestore.django import modulestore +from xmodule.modulestore.tests.django_utils import ( + ModuleStoreTestCase, + SharedModuleStoreTestCase, +) +from xmodule.modulestore.tests.factories import BlockFactory, CourseFactory User = get_user_model() @@ -274,7 +261,11 @@ def test_basic(self, mock_emit): ) self.register_post_thread_response(cs_thread) with self.assert_signal_sent( - api, "thread_created", sender=None, user=self.user, exclude_args=("post", "notify_all_learners") + api, + "thread_created", + sender=None, + user=self.user, + exclude_args=("post", "notify_all_learners"), ): actual = create_thread(self.request, self.minimal_data) expected = self.expected_thread_data( @@ -353,7 +344,11 @@ def test_basic_in_blackout_period_with_user_access(self, mock_emit): ) with self.assert_signal_sent( - api, "thread_created", sender=None, user=self.user, exclude_args=("post", "notify_all_learners") + api, + "thread_created", + sender=None, + user=self.user, + exclude_args=("post", "notify_all_learners"), ): actual = create_thread(self.request, self.minimal_data) expected = self.expected_thread_data( @@ -379,6 +374,7 @@ def test_basic_in_blackout_period_with_user_access(self, mock_emit): "type", "voted", ], + "is_deleted": False, } ) assert actual == expected @@ -430,7 +426,11 @@ def test_title_truncation(self, mock_emit): ) self.register_post_thread_response(cs_thread) with self.assert_signal_sent( - api, "thread_created", sender=None, user=self.user, exclude_args=("post", "notify_all_learners") + api, + "thread_created", + sender=None, + user=self.user, + exclude_args=("post", "notify_all_learners"), ): create_thread(self.request, data) event_name, event_data = mock_emit.call_args[0] @@ -718,6 +718,10 @@ def test_success(self, parent_id, mock_emit): "image_url_medium": "http://testserver/static/default_50.png", "image_url_small": "http://testserver/static/default_30.png", }, + "is_deleted": None, + "deleted_at": None, + "deleted_by": None, + "deleted_by_label": None, } assert actual == expected @@ -826,6 +830,10 @@ def test_success_in_black_out_with_user_access(self, parent_id, mock_emit): "image_url_medium": "http://testserver/static/default_50.png", "image_url_small": "http://testserver/static/default_30.png", }, + "is_deleted": False, + "deleted_at": None, + "deleted_by": None, + "deleted_by_label": None, } assert actual == expected @@ -914,7 +922,9 @@ def test_endorsed(self, role_name, is_thread_author, thread_type): ) try: create_comment(self.request, data) - last_commemt_params = self.get_mock_func_calls("create_parent_comment")[-1][1] + last_commemt_params = self.get_mock_func_calls("create_parent_comment")[-1][ + 1 + ] assert last_commemt_params["endorsed"] assert not expected_error except ValidationError: @@ -1828,6 +1838,10 @@ def test_basic(self, parent_id): "image_url_medium": "http://testserver/static/default_50.png", "image_url_small": "http://testserver/static/default_30.png", }, + "is_deleted": None, + "deleted_at": None, + "deleted_by": None, + "deleted_by_label": None, } assert actual == expected params = { @@ -1888,7 +1902,7 @@ def test_abuse_flagged(self, old_flagged, new_flagged, mock_emit): else "edx.forum.response.unreported" ) expected_event_data = { - "discussion": {'id': 'test_thread'}, + "discussion": {"id": "test_thread"}, "body": "Original body", "id": "test_comment", "content_type": "Response", @@ -1951,7 +1965,7 @@ def test_comment_un_abuse_flag_for_moderator_role( "body": "Original body", "id": "test_comment", "content_type": "Response", - "discussion": {'id': 'test_thread'}, + "discussion": {"id": "test_thread"}, "commentable_id": "dummy", "truncated": False, "url": "", @@ -2370,6 +2384,7 @@ def test_basic(self, mock_emit): params = { "thread_id": self.thread_id, "course_id": str(self.course.id), + "deleted_by": str(self.user.id), } self.check_mock_called_with("delete_thread", -1, **params) @@ -2557,6 +2572,7 @@ def test_basic(self, mock_emit): params = { "comment_id": self.comment_id, "course_id": str(self.course.id), + "deleted_by": str(self.user.id), } self.check_mock_called_with("delete_comment", -1, **params) @@ -2921,6 +2937,7 @@ def test_get_threads_by_topic_id(self): "page": 1, "per_page": 1, "commentable_ids": ["topic_x", "topic_meow"], + "show_deleted": False, } self.check_mock_called_with( "get_user_threads", @@ -2936,6 +2953,7 @@ def test_basic_query_params(self): "sort_key": "activity", "page": 6, "per_page": 14, + "show_deleted": False, } self.check_mock_called_with( "get_user_threads", @@ -3076,10 +3094,10 @@ def test_request_group(self, role_name, course_is_cohorted): self.get_thread_list([], course=cohort_course) thread_func_params = self.get_mock_func_calls("get_user_threads")[-1][1] actual_has_group = "group_id" in thread_func_params - expected_has_group = ( - course_is_cohorted and role_name in ( - FORUM_ROLE_STUDENT, FORUM_ROLE_COMMUNITY_TA, FORUM_ROLE_GROUP_MODERATOR - ) + expected_has_group = course_is_cohorted and role_name in ( + FORUM_ROLE_STUDENT, + FORUM_ROLE_COMMUNITY_TA, + FORUM_ROLE_GROUP_MODERATOR, ) assert actual_has_group == expected_has_group @@ -3144,6 +3162,7 @@ def test_text_search(self, text_search_rewrite): "page": 1, "per_page": 10, "text": "test search string", + "show_deleted": False, } self.check_mock_called_with( "search_threads", @@ -3170,6 +3189,7 @@ def test_filter_threads_by_author(self): "page": 1, "per_page": 10, "author_id": str(self.user.id), + "show_deleted": False, } self.check_mock_called_with( "get_user_threads", @@ -3216,6 +3236,7 @@ def test_thread_type(self, thread_type): "page": 1, "per_page": 10, "thread_type": thread_type, + "show_deleted": False, } if thread_type is None: @@ -3253,6 +3274,7 @@ def test_flagged(self, flagged_boolean): "page": 1, "per_page": 10, "flagged": flagged_boolean, + "show_deleted": False, } if flagged_boolean is None: @@ -3293,6 +3315,7 @@ def test_flagged_count(self, role): "count_flagged": True, "page": 1, "per_page": 10, + "show_deleted": False, } self.check_mock_called_with( @@ -3341,6 +3364,7 @@ def test_following(self): "sort_key": "activity", "page": 1, "per_page": 11, + "show_deleted": False, } self.check_mock_called_with("get_user_subscriptions", -1, **params) @@ -3368,6 +3392,7 @@ def test_view_query(self, query): "page": 1, "per_page": 11, query: True, + "show_deleted": False, } self.check_mock_called_with( "get_user_threads", @@ -3409,6 +3434,7 @@ def test_order_by_query(self, http_query, cc_query): "sort_key": cc_query, "page": 1, "per_page": 11, + "show_deleted": False, } self.check_mock_called_with( "get_user_threads", @@ -3441,6 +3467,7 @@ def test_order_direction(self): "sort_key": "activity", "page": 1, "per_page": 11, + "show_deleted": False, } self.check_mock_called_with( "get_user_threads", @@ -3769,6 +3796,10 @@ def get_source_and_expected_comments(self): "image_url_medium": "http://testserver/static/default_50.png", "image_url_small": "http://testserver/static/default_30.png", }, + "is_deleted": None, + "deleted_at": None, + "deleted_by": None, + "deleted_by_label": None, }, { "id": "test_comment_2", @@ -3804,6 +3835,10 @@ def get_source_and_expected_comments(self): "image_url_medium": "http://testserver/static/default_50.png", "image_url_small": "http://testserver/static/default_30.png", }, + "is_deleted": None, + "deleted_at": None, + "deleted_by": None, + "deleted_by_label": None, }, ] return source_comments, expected_comments diff --git a/lms/djangoapps/discussion/rest_api/tests/test_forms.py b/lms/djangoapps/discussion/rest_api/tests/test_forms.py index 3be65964b6b9..33359337933b 100644 --- a/lms/djangoapps/discussion/rest_api/tests/test_forms.py +++ b/lms/djangoapps/discussion/rest_api/tests/test_forms.py @@ -2,7 +2,6 @@ Tests for Discussion API forms """ - import itertools from unittest import TestCase from urllib.parse import urlencode @@ -12,9 +11,9 @@ from opaque_keys.edx.locator import CourseLocator from lms.djangoapps.discussion.rest_api.forms import ( - UserCommentListGetForm, CommentListGetForm, ThreadListGetForm, + UserCommentListGetForm, ) from openedx.core.djangoapps.util.test_forms import FormTestMixin @@ -36,7 +35,9 @@ def test_missing_page_size(self): def test_zero_page_size(self): self.form_data["page_size"] = "0" - self.assert_error("page_size", "Ensure this value is greater than or equal to 1.") + self.assert_error( + "page_size", "Ensure this value is greater than or equal to 1." + ) def test_excessive_page_size(self): self.form_data["page_size"] = "101" @@ -46,6 +47,7 @@ def test_excessive_page_size(self): @ddt.ddt class ThreadListGetFormTest(FormTestMixin, PaginationTestMixin, TestCase): """Tests for ThreadListGetForm""" + FORM_CLASS = ThreadListGetForm def setUp(self): @@ -58,37 +60,41 @@ def setUp(self): "page_size": "13", } ), - mutable=True + mutable=True, ) def test_basic(self): form = self.get_form(expected_valid=True) assert form.cleaned_data == { - 'course_id': CourseLocator.from_string('Foo/Bar/Baz'), - 'page': 2, - 'page_size': 13, - 'count_flagged': None, - 'topic_id': set(), - 'text_search': '', - 'following': None, - 'author': '', - 'thread_type': '', - 'flagged': None, - 'view': '', - 'order_by': 'last_activity_at', - 'order_direction': 'desc', - 'requested_fields': set() + "course_id": CourseLocator.from_string("Foo/Bar/Baz"), + "page": 2, + "page_size": 13, + "count_flagged": None, + "topic_id": set(), + "text_search": "", + "following": None, + "author": "", + "thread_type": "", + "flagged": None, + "show_deleted": None, + "view": "", + "order_by": "last_activity_at", + "order_direction": "desc", + "requested_fields": set(), } def test_topic_id(self): self.form_data.setlist("topic_id", ["example topic_id", "example 2nd topic_id"]) form = self.get_form(expected_valid=True) - assert form.cleaned_data['topic_id'] == {'example topic_id', 'example 2nd topic_id'} + assert form.cleaned_data["topic_id"] == { + "example topic_id", + "example 2nd topic_id", + } def test_text_search(self): self.form_data["text_search"] = "test search string" form = self.get_form(expected_valid=True) - assert form.cleaned_data['text_search'] == 'test search string' + assert form.cleaned_data["text_search"] == "test search string" def test_missing_course_id(self): self.form_data.pop("course_id") @@ -109,7 +115,10 @@ def test_thread_type(self, value): def test_thread_type_invalid(self): self.form_data["thread_type"] = "invalid-option" - self.assert_error("thread_type", "Select a valid choice. invalid-option is not one of the available choices.") + self.assert_error( + "thread_type", + "Select a valid choice. invalid-option is not one of the available choices.", + ) @ddt.data("True", "true", 1, True) def test_flagged_true(self, value): @@ -133,7 +142,9 @@ def test_following_true(self, value): @ddt.data("False", "false", 0, False) def test_following_false(self, value): self.form_data["following"] = value - self.assert_error("following", "The value of the 'following' parameter must be true.") + self.assert_error( + "following", "The value of the 'following' parameter must be true." + ) def test_invalid_following(self): self.form_data["following"] = "invalid-boolean" @@ -144,25 +155,28 @@ def test_mutually_exclusive(self, params): self.form_data.update({param: "True" for param in params}) self.assert_error( "__all__", - "The following query parameters are mutually exclusive: topic_id, text_search, following" + "The following query parameters are mutually exclusive: topic_id, text_search, following", ) def test_invalid_view_choice(self): self.form_data["view"] = "not_a_valid_choice" - self.assert_error("view", "Select a valid choice. not_a_valid_choice is not one of the available choices.") + self.assert_error( + "view", + "Select a valid choice. not_a_valid_choice is not one of the available choices.", + ) def test_invalid_sort_by_choice(self): self.form_data["order_by"] = "not_a_valid_choice" self.assert_error( "order_by", - "Select a valid choice. not_a_valid_choice is not one of the available choices." + "Select a valid choice. not_a_valid_choice is not one of the available choices.", ) def test_invalid_sort_direction_choice(self): self.form_data["order_direction"] = "not_a_valid_choice" self.assert_error( "order_direction", - "Select a valid choice. not_a_valid_choice is not one of the available choices." + "Select a valid choice. not_a_valid_choice is not one of the available choices.", ) @ddt.data( @@ -181,12 +195,13 @@ def test_valid_choice_fields(self, field, value): def test_requested_fields(self): self.form_data["requested_fields"] = "profile_image" form = self.get_form(expected_valid=True) - assert form.cleaned_data['requested_fields'] == {'profile_image'} + assert form.cleaned_data["requested_fields"] == {"profile_image"} @ddt.ddt class CommentListGetFormTest(FormTestMixin, PaginationTestMixin, TestCase): """Tests for CommentListGetForm""" + FORM_CLASS = CommentListGetForm def setUp(self): @@ -202,13 +217,14 @@ def setUp(self): def test_basic(self): form = self.get_form(expected_valid=True) assert form.cleaned_data == { - 'thread_id': 'deadbeef', - 'endorsed': False, - 'page': 2, - 'page_size': 13, - 'flagged': False, - 'requested_fields': set(), - 'merge_question_type_responses': False + "thread_id": "deadbeef", + "endorsed": False, + "page": 2, + "page_size": 13, + "flagged": False, + "requested_fields": set(), + "merge_question_type_responses": False, + "show_deleted": None, } def test_missing_thread_id(self): @@ -236,12 +252,13 @@ def test_invalid_endorsed(self): def test_requested_fields(self): self.form_data["requested_fields"] = {"profile_image"} form = self.get_form(expected_valid=True) - assert form.cleaned_data['requested_fields'] == {'profile_image'} + assert form.cleaned_data["requested_fields"] == {"profile_image"} @ddt.ddt class UserCommentListGetFormTest(FormTestMixin, PaginationTestMixin, TestCase): """Tests for UserCommentListGetForm""" + FORM_CLASS = UserCommentListGetForm def setUp(self): @@ -256,11 +273,11 @@ def setUp(self): def test_basic(self): form = self.get_form(expected_valid=True) assert form.cleaned_data == { - 'course_id': CourseLocator.from_string('a/b/c'), - 'flagged': False, - 'page': 2, - 'page_size': 13, - 'requested_fields': set() + "course_id": CourseLocator.from_string("a/b/c"), + "flagged": False, + "page": 2, + "page_size": 13, + "requested_fields": set(), } def test_missing_flagged(self): @@ -280,7 +297,7 @@ def test_flagged_true(self, value): def test_requested_fields(self): self.form_data["requested_fields"] = {"profile_image"} form = self.get_form(expected_valid=True) - assert form.cleaned_data['requested_fields'] == {'profile_image'} + assert form.cleaned_data["requested_fields"] == {"profile_image"} def test_missing_course_id(self): self.form_data.pop("course_id") diff --git a/lms/djangoapps/discussion/rest_api/tests/test_serializers.py b/lms/djangoapps/discussion/rest_api/tests/test_serializers.py index a1443252a1ce..10f9b7a64248 100644 --- a/lms/djangoapps/discussion/rest_api/tests/test_serializers.py +++ b/lms/djangoapps/discussion/rest_api/tests/test_serializers.py @@ -9,19 +9,17 @@ import httpretty from django.test.client import RequestFactory from django.test.utils import override_settings -from xmodule.modulestore import ModuleStoreEnum -from xmodule.modulestore.django import modulestore -from xmodule.modulestore.tests.django_utils import SharedModuleStoreTestCase -from xmodule.modulestore.tests.factories import CourseFactory from common.djangoapps.student.tests.factories import UserFactory from common.djangoapps.util.testing import UrlResetMixin -from lms.djangoapps.discussion.django_comment_client.tests.utils import ForumsEnableMixin +from lms.djangoapps.discussion.django_comment_client.tests.utils import ( + ForumsEnableMixin, +) from lms.djangoapps.discussion.rest_api.serializers import ( CommentSerializer, ThreadSerializer, filter_spam_urls_from_html, - get_context + get_context, ) from lms.djangoapps.discussion.rest_api.tests.utils import ( CommentsServiceMockMixin, @@ -39,6 +37,10 @@ FORUM_ROLE_STUDENT, Role, ) +from xmodule.modulestore import ModuleStoreEnum +from xmodule.modulestore.django import modulestore +from xmodule.modulestore.tests.django_utils import SharedModuleStoreTestCase +from xmodule.modulestore.tests.factories import CourseFactory @ddt.ddt @@ -46,13 +48,18 @@ class SerializerTestMixin(ForumsEnableMixin, CommentsServiceMockMixin, UrlResetM """ Test Mixin for Serializer tests """ + @classmethod - @mock.patch.dict("django.conf.settings.FEATURES", {"ENABLE_DISCUSSION_SERVICE": True}) + @mock.patch.dict( + "django.conf.settings.FEATURES", {"ENABLE_DISCUSSION_SERVICE": True} + ) def setUpClass(cls): super().setUpClass() cls.course = CourseFactory.create() - @mock.patch.dict("django.conf.settings.FEATURES", {"ENABLE_DISCUSSION_SERVICE": True}) + @mock.patch.dict( + "django.conf.settings.FEATURES", {"ENABLE_DISCUSSION_SERVICE": True} + ) def setUp(self): super().setUp() httpretty.reset() @@ -60,8 +67,8 @@ def setUp(self): self.addCleanup(httpretty.reset) self.addCleanup(httpretty.disable) patcher = mock.patch( - 'openedx.core.djangoapps.discussions.config.waffle.ENABLE_FORUM_V2.is_enabled', - return_value=False + "openedx.core.djangoapps.discussions.config.waffle.ENABLE_FORUM_V2.is_enabled", + return_value=False, ) patcher.start() self.addCleanup(patcher.stop) @@ -89,7 +96,9 @@ def create_role(self, role_name, users, course=None): (FORUM_ROLE_STUDENT, False, True, True), ) @ddt.unpack - def test_anonymity(self, role_name, anonymous, anonymous_to_peers, expected_serialized_anonymous): + def test_anonymity( + self, role_name, anonymous, anonymous_to_peers, expected_serialized_anonymous + ): """ Test that content is properly made anonymous. @@ -107,7 +116,9 @@ def test_anonymity(self, role_name, anonymous, anonymous_to_peers, expected_seri """ self.create_role(role_name, [self.user]) serialized = self.serialize( - self.make_cs_content({"anonymous": anonymous, "anonymous_to_peers": anonymous_to_peers}) + self.make_cs_content( + {"anonymous": anonymous, "anonymous_to_peers": anonymous_to_peers} + ) ) actual_serialized_anonymous = serialized["author"] is None assert actual_serialized_anonymous == expected_serialized_anonymous @@ -138,17 +149,19 @@ def test_author_labels(self, role_name, anonymous, expected_label): """ self.create_role(role_name, [self.author]) serialized = self.serialize(self.make_cs_content({"anonymous": anonymous})) - assert serialized['author_label'] == expected_label + assert serialized["author_label"] == expected_label def test_abuse_flagged(self): - serialized = self.serialize(self.make_cs_content({"abuse_flaggers": [str(self.user.id)]})) - assert serialized['abuse_flagged'] is True + serialized = self.serialize( + self.make_cs_content({"abuse_flaggers": [str(self.user.id)]}) + ) + assert serialized["abuse_flagged"] is True def test_voted(self): thread_id = "test_thread" self.register_get_user_response(self.user, upvoted_ids=[thread_id]) serialized = self.serialize(self.make_cs_content({"id": thread_id})) - assert serialized['voted'] is True + assert serialized["voted"] is True @ddt.ddt @@ -175,47 +188,61 @@ def serialize(self, thread): Create a serializer with an appropriate context and use it to serialize the given thread, returning the result. """ - return ThreadSerializer(thread, context=get_context(self.course, self.request)).data + return ThreadSerializer( + thread, context=get_context(self.course, self.request) + ).data def test_basic(self): - thread = make_minimal_cs_thread({ - "id": "test_thread", - "course_id": str(self.course.id), - "commentable_id": "test_topic", - "user_id": str(self.author.id), - "username": self.author.username, - "title": "Test Title", - "body": "Test body", - "pinned": True, - "votes": {"up_count": 4}, - "comments_count": 5, - "unread_comments_count": 3, - }) - expected = self.expected_thread_data({ - "author": self.author.username, - "can_delete": False, - "vote_count": 4, - "comment_count": 6, - "unread_comment_count": 3, - "pinned": True, - "editable_fields": ["abuse_flagged", "copy_link", "following", "read", "voted"], - "abuse_flagged_count": None, - "edit_by_label": None, - "closed_by_label": None, - }) + thread = make_minimal_cs_thread( + { + "id": "test_thread", + "course_id": str(self.course.id), + "commentable_id": "test_topic", + "user_id": str(self.author.id), + "username": self.author.username, + "title": "Test Title", + "body": "Test body", + "pinned": True, + "votes": {"up_count": 4}, + "comments_count": 5, + "unread_comments_count": 3, + } + ) + expected = self.expected_thread_data( + { + "author": self.author.username, + "can_delete": False, + "vote_count": 4, + "comment_count": 6, + "unread_comment_count": 3, + "pinned": True, + "editable_fields": [ + "abuse_flagged", + "copy_link", + "following", + "read", + "voted", + ], + "abuse_flagged_count": None, + "edit_by_label": None, + "closed_by_label": None, + } + ) assert self.serialize(thread) == expected thread["thread_type"] = "question" - expected.update({ - "type": "question", - "comment_list_url": None, - "endorsed_comment_list_url": ( - "http://testserver/api/discussion/v1/comments/?thread_id=test_thread&endorsed=True" - ), - "non_endorsed_comment_list_url": ( - "http://testserver/api/discussion/v1/comments/?thread_id=test_thread&endorsed=False" - ), - }) + expected.update( + { + "type": "question", + "comment_list_url": None, + "endorsed_comment_list_url": ( + "http://testserver/api/discussion/v1/comments/?thread_id=test_thread&endorsed=True" + ), + "non_endorsed_comment_list_url": ( + "http://testserver/api/discussion/v1/comments/?thread_id=test_thread&endorsed=False" + ), + } + ) assert self.serialize(thread) == expected def test_pinned_missing(self): @@ -227,34 +254,34 @@ def test_pinned_missing(self): del thread_data["pinned"] self.register_get_thread_response(thread_data) serialized = self.serialize(thread_data) - assert serialized['pinned'] is False + assert serialized["pinned"] is False def test_group(self): self.course.cohort_config = {"cohorted": True} modulestore().update_item(self.course, ModuleStoreEnum.UserID.test) cohort = CohortFactory.create(course_id=self.course.id) serialized = self.serialize(self.make_cs_content({"group_id": cohort.id})) - assert serialized['group_id'] == cohort.id - assert serialized['group_name'] == cohort.name + assert serialized["group_id"] == cohort.id + assert serialized["group_name"] == cohort.name def test_following(self): thread_id = "test_thread" self.register_get_user_response(self.user, subscribed_thread_ids=[thread_id]) serialized = self.serialize(self.make_cs_content({"id": thread_id})) - assert serialized['following'] is True + assert serialized["following"] is True def test_response_count(self): thread_data = self.make_cs_content({"resp_total": 2}) self.register_get_thread_response(thread_data) serialized = self.serialize(thread_data) - assert serialized['response_count'] == 2 + assert serialized["response_count"] == 2 def test_response_count_missing(self): thread_data = self.make_cs_content({}) del thread_data["resp_total"] self.register_get_thread_response(thread_data) serialized = self.serialize(thread_data) - assert 'response_count' not in serialized + assert "response_count" not in serialized @ddt.data( (FORUM_ROLE_MODERATOR, True), @@ -272,43 +299,62 @@ def test_closed_by_label_field(self, role, visible): self.create_role(FORUM_ROLE_MODERATOR, [moderator]) self.create_role(request_role, [self.user]) - thread = make_minimal_cs_thread({ - "id": "test_thread", - "course_id": str(self.course.id), - "commentable_id": "test_topic", - "user_id": str(author.id), - "username": author.username, - "title": "Test Title", - "body": "Test body", - "pinned": True, - "votes": {"up_count": 4}, - "comments_count": 5, - "unread_comments_count": 3, - "closed_by": moderator - }) + thread = make_minimal_cs_thread( + { + "id": "test_thread", + "course_id": str(self.course.id), + "commentable_id": "test_topic", + "user_id": str(author.id), + "username": author.username, + "title": "Test Title", + "body": "Test body", + "pinned": True, + "votes": {"up_count": 4}, + "comments_count": 5, + "unread_comments_count": 3, + "closed_by": moderator, + } + ) closed_by_label = "Moderator" if visible else None closed_by = moderator if visible else None can_delete = role != FORUM_ROLE_STUDENT editable_fields = ["abuse_flagged", "copy_link", "following", "read", "voted"] if role == "author": editable_fields.remove("voted") - editable_fields.extend(['anonymous', 'raw_body', 'title', 'topic_id', 'type']) + editable_fields.extend( + ["anonymous", "raw_body", "title", "topic_id", "type"] + ) elif role == FORUM_ROLE_MODERATOR: - editable_fields.extend(['close_reason_code', 'closed', 'edit_reason_code', 'pinned', - 'raw_body', 'title', 'topic_id', 'type']) - expected = self.expected_thread_data({ - "author": author.username, - "can_delete": can_delete, - "vote_count": 4, - "comment_count": 6, - "unread_comment_count": 3, - "pinned": True, - "editable_fields": sorted(editable_fields), - "abuse_flagged_count": None, - "edit_by_label": None, - "closed_by_label": closed_by_label, - "closed_by": closed_by, - }) + editable_fields.extend( + [ + "close_reason_code", + "closed", + "edit_reason_code", + "pinned", + "raw_body", + "title", + "topic_id", + "type", + ] + ) + # is_deleted is visible (False) for privileged users, hidden (None) for others + is_deleted = False if role == FORUM_ROLE_MODERATOR else None + expected = self.expected_thread_data( + { + "author": author.username, + "can_delete": can_delete, + "vote_count": 4, + "comment_count": 6, + "unread_comment_count": 3, + "pinned": True, + "editable_fields": sorted(editable_fields), + "abuse_flagged_count": None, + "edit_by_label": None, + "closed_by_label": closed_by_label, + "closed_by": closed_by, + "is_deleted": is_deleted, + } + ) assert self.serialize(thread) == expected @ddt.data( @@ -327,48 +373,69 @@ def test_edit_by_label_field(self, role, visible): self.create_role(FORUM_ROLE_MODERATOR, [moderator]) self.create_role(request_role, [self.user]) - thread = make_minimal_cs_thread({ - "id": "test_thread", - "course_id": str(self.course.id), - "commentable_id": "test_topic", - "user_id": str(author.id), - "username": author.username, - "title": "Test Title", - "body": "Test body", - "pinned": True, - "votes": {"up_count": 4}, - "edit_history": [{"editor_username": moderator}], - "comments_count": 5, - "unread_comments_count": 3, - "closed_by": None - }) + thread = make_minimal_cs_thread( + { + "id": "test_thread", + "course_id": str(self.course.id), + "commentable_id": "test_topic", + "user_id": str(author.id), + "username": author.username, + "title": "Test Title", + "body": "Test body", + "pinned": True, + "votes": {"up_count": 4}, + "edit_history": [{"editor_username": moderator}], + "comments_count": 5, + "unread_comments_count": 3, + "closed_by": None, + } + ) edit_by_label = "Moderator" if visible else None can_delete = role != FORUM_ROLE_STUDENT - last_edit = None if role == FORUM_ROLE_STUDENT else {"editor_username": moderator} + last_edit = ( + None if role == FORUM_ROLE_STUDENT else {"editor_username": moderator} + ) editable_fields = ["abuse_flagged", "copy_link", "following", "read", "voted"] if role == "author": editable_fields.remove("voted") - editable_fields.extend(['anonymous', 'raw_body', 'title', 'topic_id', 'type']) + editable_fields.extend( + ["anonymous", "raw_body", "title", "topic_id", "type"] + ) elif role == FORUM_ROLE_MODERATOR: - editable_fields.extend(['close_reason_code', 'closed', 'edit_reason_code', 'pinned', - 'raw_body', 'title', 'topic_id', 'type']) + editable_fields.extend( + [ + "close_reason_code", + "closed", + "edit_reason_code", + "pinned", + "raw_body", + "title", + "topic_id", + "type", + ] + ) - expected = self.expected_thread_data({ - "author": author.username, - "can_delete": can_delete, - "vote_count": 4, - "comment_count": 6, - "unread_comment_count": 3, - "pinned": True, - "editable_fields": sorted(editable_fields), - "abuse_flagged_count": None, - "last_edit": last_edit, - "edit_by_label": edit_by_label, - "closed_by_label": None, - "closed_by": None, - }) + # is_deleted is visible (False) for privileged users, hidden (None) for others + is_deleted = False if role == FORUM_ROLE_MODERATOR else None + expected = self.expected_thread_data( + { + "author": author.username, + "can_delete": can_delete, + "vote_count": 4, + "comment_count": 6, + "unread_comment_count": 3, + "pinned": True, + "editable_fields": sorted(editable_fields), + "abuse_flagged_count": None, + "last_edit": last_edit, + "edit_by_label": edit_by_label, + "closed_by_label": None, + "closed_by": None, + "is_deleted": is_deleted, + } + ) assert self.serialize(thread) == expected def test_get_preview_body(self): @@ -384,7 +451,10 @@ def test_get_preview_body(self): {"body": "

    This is a test thread body with some text.

    "} ) serialized = self.serialize(thread_data) - assert serialized['preview_body'] == "This is a test thread body with some text." + assert ( + serialized["preview_body"] + == "This is a test thread body with some text." + ) @ddt.ddt @@ -402,12 +472,12 @@ def make_cs_content(self, overrides=None, with_endorsement=False): """ merged_overrides = { "user_id": str(self.author.id), - "username": self.author.username + "username": self.author.username, } if with_endorsement: merged_overrides["endorsement"] = { "user_id": str(self.endorser.id), - "time": self.endorsed_at + "time": self.endorsed_at, } merged_overrides.update(overrides or {}) return make_minimal_cs_comment(merged_overrides) @@ -417,7 +487,9 @@ def serialize(self, comment, thread_data=None): Create a serializer with an appropriate context and use it to serialize the given comment, returning the result. """ - context = get_context(self.course, self.request, make_minimal_cs_thread(thread_data)) + context = get_context( + self.course, self.request, make_minimal_cs_thread(thread_data) + ) return CommentSerializer(comment, context=context).data def test_basic(self): @@ -472,6 +544,10 @@ def test_basic(self): "image_url_medium": "http://testserver/static/default_50.png", "image_url_small": "http://testserver/static/default_30.png", }, + "is_deleted": None, + "deleted_at": None, + "deleted_by": None, + "deleted_by_label": None, } assert self.serialize(comment) == expected @@ -484,7 +560,7 @@ def test_basic(self): FORUM_ROLE_COMMUNITY_TA, FORUM_ROLE_STUDENT, ], - [True, False] + [True, False], ) ) @ddt.unpack @@ -501,10 +577,12 @@ def test_endorsed_by(self, endorser_role_name, thread_anonymous): self.create_role(endorser_role_name, [self.endorser]) serialized = self.serialize( self.make_cs_content(with_endorsement=True), - thread_data={"anonymous": thread_anonymous} + thread_data={"anonymous": thread_anonymous}, ) actual_endorser_anonymous = serialized["endorsed_by"] is None - expected_endorser_anonymous = endorser_role_name == FORUM_ROLE_STUDENT and thread_anonymous + expected_endorser_anonymous = ( + endorser_role_name == FORUM_ROLE_STUDENT and thread_anonymous + ) assert actual_endorser_anonymous == expected_endorser_anonymous @ddt.data( @@ -527,56 +605,69 @@ def test_endorsed_by_labels(self, role_name, expected_label): """ self.create_role(role_name, [self.endorser]) serialized = self.serialize(self.make_cs_content(with_endorsement=True)) - assert serialized['endorsed_by_label'] == expected_label + assert serialized["endorsed_by_label"] == expected_label def test_endorsed_at(self): serialized = self.serialize(self.make_cs_content(with_endorsement=True)) - assert serialized['endorsed_at'] == self.endorsed_at + assert serialized["endorsed_at"] == self.endorsed_at def test_children(self): - comment = self.make_cs_content({ - "id": "test_root", - "children": [ - self.make_cs_content({ - "id": "test_child_1", - "parent_id": "test_root", - }), - self.make_cs_content({ - "id": "test_child_2", - "parent_id": "test_root", - "children": [ - self.make_cs_content({ - "id": "test_grandchild", - "parent_id": "test_child_2" - }) - ], - }), - ], - }) + comment = self.make_cs_content( + { + "id": "test_root", + "children": [ + self.make_cs_content( + { + "id": "test_child_1", + "parent_id": "test_root", + } + ), + self.make_cs_content( + { + "id": "test_child_2", + "parent_id": "test_root", + "children": [ + self.make_cs_content( + { + "id": "test_grandchild", + "parent_id": "test_child_2", + } + ) + ], + } + ), + ], + } + ) serialized = self.serialize(comment) - assert serialized['children'][0]['id'] == 'test_child_1' - assert serialized['children'][0]['parent_id'] == 'test_root' - assert serialized['children'][1]['id'] == 'test_child_2' - assert serialized['children'][1]['parent_id'] == 'test_root' - assert serialized['children'][1]['children'][0]['id'] == 'test_grandchild' - assert serialized['children'][1]['children'][0]['parent_id'] == 'test_child_2' + assert serialized["children"][0]["id"] == "test_child_1" + assert serialized["children"][0]["parent_id"] == "test_root" + assert serialized["children"][1]["id"] == "test_child_2" + assert serialized["children"][1]["parent_id"] == "test_root" + assert serialized["children"][1]["children"][0]["id"] == "test_grandchild" + assert serialized["children"][1]["children"][0]["parent_id"] == "test_child_2" @ddt.ddt class ThreadSerializerDeserializationTest( - ForumsEnableMixin, - CommentsServiceMockMixin, - UrlResetMixin, - SharedModuleStoreTestCase + ForumsEnableMixin, + CommentsServiceMockMixin, + UrlResetMixin, + SharedModuleStoreTestCase, ): """Tests for ThreadSerializer deserialization.""" + @classmethod - @mock.patch.dict("django.conf.settings.FEATURES", {"ENABLE_DISCUSSION_SERVICE": True}) + @mock.patch.dict( + "django.conf.settings.FEATURES", {"ENABLE_DISCUSSION_SERVICE": True} + ) def setUpClass(cls): super().setUpClass() cls.course = CourseFactory.create() - @mock.patch.dict("django.conf.settings.FEATURES", {"ENABLE_DISCUSSION_SERVICE": True}) + @mock.patch.dict( + "django.conf.settings.FEATURES", {"ENABLE_DISCUSSION_SERVICE": True} + ) def setUp(self): super().setUp() httpretty.reset() @@ -584,8 +675,8 @@ def setUp(self): self.addCleanup(httpretty.reset) self.addCleanup(httpretty.disable) patcher = mock.patch( - 'openedx.core.djangoapps.discussions.config.waffle.ENABLE_FORUM_V2.is_enabled', - return_value=False + "openedx.core.djangoapps.discussions.config.waffle.ENABLE_FORUM_V2.is_enabled", + return_value=False, ) patcher.start() self.addCleanup(patcher.stop) @@ -600,18 +691,22 @@ def setUp(self): "title": "Test Title", "raw_body": "Test body", } - self.existing_thread = Thread(**make_minimal_cs_thread({ - "id": "existing_thread", - "course_id": str(self.course.id), - "commentable_id": "original_topic", - "thread_type": "discussion", - "title": "Original Title", - "body": "Original body", - "user_id": str(self.user.id), - "username": self.user.username, - "read": "False", - "endorsed": "False" - })) + self.existing_thread = Thread( + **make_minimal_cs_thread( + { + "id": "existing_thread", + "course_id": str(self.course.id), + "commentable_id": "original_topic", + "thread_type": "discussion", + "title": "Original Title", + "body": "Original body", + "user_id": str(self.user.id), + "username": self.user.username, + "read": "False", + "endorsed": "False", + } + ) + ) def save_and_reserialize(self, data, instance=None): """ @@ -623,7 +718,7 @@ def save_and_reserialize(self, data, instance=None): instance, data=data, partial=(instance is not None), - context=get_context(self.course, self.request) + context=get_context(self.course, self.request), ) assert serializer.is_valid() serializer.save() @@ -635,33 +730,36 @@ def test_create_missing_field(self): data.pop(field) serializer = ThreadSerializer(data=data) assert not serializer.is_valid() - assert serializer.errors == {field: ['This field is required.']} + assert serializer.errors == {field: ["This field is required."]} @ddt.data("", " ") def test_create_empty_string(self, value): data = self.minimal_data.copy() data.update({field: value for field in ["topic_id", "title", "raw_body"]}) - serializer = ThreadSerializer(data=data, context=get_context(self.course, self.request)) + serializer = ThreadSerializer( + data=data, context=get_context(self.course, self.request) + ) assert not serializer.is_valid() assert serializer.errors == { - field: ['This field may not be blank.'] for field in ['topic_id', 'title', 'raw_body'] + field: ["This field may not be blank."] + for field in ["topic_id", "title", "raw_body"] } def test_update_empty(self): self.register_put_thread_response(self.existing_thread.attributes) self.save_and_reserialize({}, self.existing_thread) assert parsed_body(httpretty.last_request()) == { - 'course_id': [str(self.course.id)], - 'commentable_id': ['original_topic'], - 'thread_type': ['discussion'], - 'title': ['Original Title'], - 'body': ['Original body'], - 'anonymous': ['False'], - 'anonymous_to_peers': ['False'], - 'closed': ['False'], - 'pinned': ['False'], - 'user_id': [str(self.user.id)], - 'read': ['False'] + "course_id": [str(self.course.id)], + "commentable_id": ["original_topic"], + "thread_type": ["discussion"], + "title": ["Original Title"], + "body": ["Original body"], + "anonymous": ["False"], + "anonymous_to_peers": ["False"], + "closed": ["False"], + "pinned": ["False"], + "user_id": [str(self.user.id)], + "read": ["False"], } @ddt.data(True, False) @@ -676,18 +774,18 @@ def test_update_all(self, read): } saved = self.save_and_reserialize(data, self.existing_thread) assert parsed_body(httpretty.last_request()) == { - 'course_id': [str(self.course.id)], - 'commentable_id': ['edited_topic'], - 'thread_type': ['question'], - 'title': ['Edited Title'], - 'body': ['Edited body'], - 'anonymous': ['False'], - 'anonymous_to_peers': ['False'], - 'closed': ['False'], - 'pinned': ['False'], - 'user_id': [str(self.user.id)], - 'read': [str(read)], - 'editing_user_id': [str(self.user.id)], + "course_id": [str(self.course.id)], + "commentable_id": ["edited_topic"], + "thread_type": ["question"], + "title": ["Edited Title"], + "body": ["Edited body"], + "anonymous": ["False"], + "anonymous_to_peers": ["False"], + "closed": ["False"], + "pinned": ["False"], + "user_id": [str(self.user.id)], + "read": [str(read)], + "editing_user_id": [str(self.user.id)], } for key in data: assert saved[key] == data[key] @@ -702,7 +800,7 @@ def test_update_anonymous(self): "anonymous": True, } self.save_and_reserialize(data, self.existing_thread) - assert parsed_body(httpretty.last_request())["anonymous"] == ['True'] + assert parsed_body(httpretty.last_request())["anonymous"] == ["True"] def test_update_anonymous_to_peers(self): """ @@ -714,7 +812,7 @@ def test_update_anonymous_to_peers(self): "anonymous_to_peers": True, } self.save_and_reserialize(data, self.existing_thread) - assert parsed_body(httpretty.last_request())["anonymous_to_peers"] == ['True'] + assert parsed_body(httpretty.last_request())["anonymous_to_peers"] == ["True"] @ddt.data("", " ") def test_update_empty_string(self, value): @@ -722,11 +820,12 @@ def test_update_empty_string(self, value): self.existing_thread, data={field: value for field in ["topic_id", "title", "raw_body"]}, partial=True, - context=get_context(self.course, self.request) + context=get_context(self.course, self.request), ) assert not serializer.is_valid() assert serializer.errors == { - field: ['This field may not be blank.'] for field in ['topic_id', 'title', 'raw_body'] + field: ["This field may not be blank."] + for field in ["topic_id", "title", "raw_body"] } def test_update_course_id(self): @@ -734,15 +833,20 @@ def test_update_course_id(self): self.existing_thread, data={"course_id": "some/other/course"}, partial=True, - context=get_context(self.course, self.request) + context=get_context(self.course, self.request), ) assert not serializer.is_valid() - assert serializer.errors == {'course_id': ['This field is not allowed in an update.']} + assert serializer.errors == { + "course_id": ["This field is not allowed in an update."] + } @ddt.ddt -class CommentSerializerDeserializationTest(ForumsEnableMixin, CommentsServiceMockMixin, SharedModuleStoreTestCase): +class CommentSerializerDeserializationTest( + ForumsEnableMixin, CommentsServiceMockMixin, SharedModuleStoreTestCase +): """Tests for ThreadSerializer deserialization.""" + @classmethod def setUpClass(cls): super().setUpClass() @@ -755,8 +859,8 @@ def setUp(self): self.addCleanup(httpretty.reset) self.addCleanup(httpretty.disable) patcher = mock.patch( - 'openedx.core.djangoapps.discussions.config.waffle.ENABLE_FORUM_V2.is_enabled', - return_value=False + "openedx.core.djangoapps.discussions.config.waffle.ENABLE_FORUM_V2.is_enabled", + return_value=False, ) patcher.start() self.addCleanup(patcher.stop) @@ -778,14 +882,18 @@ def setUp(self): "thread_id": "test_thread", "raw_body": "Test body", } - self.existing_comment = Comment(**make_minimal_cs_comment({ - "id": "existing_comment", - "thread_id": "dummy", - "body": "Original body", - "user_id": str(self.user.id), - "username": self.user.username, - "course_id": str(self.course.id), - })) + self.existing_comment = Comment( + **make_minimal_cs_comment( + { + "id": "existing_comment", + "thread_id": "dummy", + "body": "Original body", + "user_id": str(self.user.id), + "username": self.user.username, + "course_id": str(self.course.id), + } + ) + ) def save_and_reserialize(self, data, instance=None): """ @@ -795,13 +903,10 @@ def save_and_reserialize(self, data, instance=None): context = get_context( self.course, self.request, - make_minimal_cs_thread({"course_id": str(self.course.id)}) + make_minimal_cs_thread({"course_id": str(self.course.id)}), ) serializer = CommentSerializer( - instance, - data=data, - partial=(instance is not None), - context=context + instance, data=data, partial=(instance is not None), context=context ) assert serializer.is_valid() serializer.save() @@ -813,21 +918,23 @@ def test_create_missing_field(self): data.pop(field) serializer = CommentSerializer( data=data, - context=get_context(self.course, self.request, make_minimal_cs_thread()) + context=get_context( + self.course, self.request, make_minimal_cs_thread() + ), ) assert not serializer.is_valid() - assert serializer.errors == {field: ['This field is required.']} + assert serializer.errors == {field: ["This field is required."]} def test_update_empty(self): self.register_put_comment_response(self.existing_comment.attributes) self.save_and_reserialize({}, instance=self.existing_comment) assert parsed_body(httpretty.last_request()) == { - 'body': ['Original body'], - 'course_id': [str(self.course.id)], - 'user_id': [str(self.user.id)], - 'anonymous': ['False'], - 'anonymous_to_peers': ['False'], - 'endorsed': ['False'] + "body": ["Original body"], + "course_id": [str(self.course.id)], + "user_id": [str(self.user.id)], + "anonymous": ["False"], + "anonymous_to_peers": ["False"], + "endorsed": ["False"], } def test_update_anonymous(self): @@ -840,7 +947,7 @@ def test_update_anonymous(self): "anonymous": True, } self.save_and_reserialize(data, self.existing_comment) - assert parsed_body(httpretty.last_request())["anonymous"] == ['True'] + assert parsed_body(httpretty.last_request())["anonymous"] == ["True"] def test_update_anonymous_to_peers(self): """ @@ -852,7 +959,7 @@ def test_update_anonymous_to_peers(self): "anonymous_to_peers": True, } self.save_and_reserialize(data, self.existing_comment) - assert parsed_body(httpretty.last_request())["anonymous_to_peers"] == ['True'] + assert parsed_body(httpretty.last_request())["anonymous_to_peers"] == ["True"] @ddt.data("thread_id", "parent_id") def test_update_non_updatable(self, field): @@ -860,23 +967,26 @@ def test_update_non_updatable(self, field): self.existing_comment, data={field: "different_value"}, partial=True, - context=get_context(self.course, self.request) + context=get_context(self.course, self.request), ) assert not serializer.is_valid() - assert serializer.errors == {field: ['This field is not allowed in an update.']} + assert serializer.errors == {field: ["This field is not allowed in an update."]} class FilterSpamTest(SharedModuleStoreTestCase): """ Tests for the filter_spam method """ - @override_settings(DISCUSSION_SPAM_URLS=['example.com']) + + @override_settings(DISCUSSION_SPAM_URLS=["example.com"]) def test_filter(self): self.assertEqual( - filter_spam_urls_from_html('')[0], - '
    abc
    ' + filter_spam_urls_from_html( + '' + )[0], + "
    abc
    ", ) self.assertEqual( - filter_spam_urls_from_html('
    example.com/abc/def
    ')[0], - '
    ' + filter_spam_urls_from_html("
    example.com/abc/def
    ")[0], + "
    ", ) diff --git a/lms/djangoapps/discussion/rest_api/tests/test_views.py b/lms/djangoapps/discussion/rest_api/tests/test_views.py index e4d46168c46d..8a9405076c2d 100644 --- a/lms/djangoapps/discussion/rest_api/tests/test_views.py +++ b/lms/djangoapps/discussion/rest_api/tests/test_views.py @@ -2,7 +2,6 @@ Tests for Discussion API views """ - import json import random from datetime import datetime @@ -20,22 +19,22 @@ from rest_framework import status from rest_framework.test import APIClient, APITestCase -from lms.djangoapps.discussion.toggles import ENABLE_DISCUSSIONS_MFE -from lms.djangoapps.discussion.rest_api.utils import get_usernames_from_search_string -from xmodule.modulestore import ModuleStoreEnum -from xmodule.modulestore.django import modulestore -from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase, SharedModuleStoreTestCase -from xmodule.modulestore.tests.factories import CourseFactory, BlockFactory, check_mongo_calls - from common.djangoapps.course_modes.models import CourseMode from common.djangoapps.course_modes.tests.factories import CourseModeFactory -from common.djangoapps.student.models import get_retired_username_by_username, CourseEnrollment -from common.djangoapps.student.roles import CourseInstructorRole, CourseStaffRole, GlobalStaff +from common.djangoapps.student.models import ( + CourseEnrollment, + get_retired_username_by_username, +) +from common.djangoapps.student.roles import ( + CourseInstructorRole, + CourseStaffRole, + GlobalStaff, +) from common.djangoapps.student.tests.factories import ( AdminFactory, CourseEnrollmentFactory, SuperuserFactory, - UserFactory + UserFactory, ) from common.djangoapps.util.testing import UrlResetMixin from lms.djangoapps.discussion.django_comment_client.tests.utils import ( @@ -48,21 +47,50 @@ make_minimal_cs_comment, make_minimal_cs_thread, ) +from lms.djangoapps.discussion.rest_api.utils import get_usernames_from_search_string +from lms.djangoapps.discussion.toggles import ENABLE_DISCUSSIONS_MFE from openedx.core.djangoapps.course_groups.tests.helpers import config_course_cohorts -from openedx.core.djangoapps.discussions.config.waffle import ENABLE_NEW_STRUCTURE_DISCUSSIONS -from openedx.core.djangoapps.discussions.models import DiscussionsConfiguration, DiscussionTopicLink, Provider -from openedx.core.djangoapps.discussions.tasks import update_discussions_settings_from_course_task +from openedx.core.djangoapps.discussions.config.waffle import ( + ENABLE_NEW_STRUCTURE_DISCUSSIONS, +) +from openedx.core.djangoapps.discussions.models import ( + DiscussionsConfiguration, + DiscussionTopicLink, + Provider, +) +from openedx.core.djangoapps.discussions.tasks import ( + update_discussions_settings_from_course_task, +) from openedx.core.djangoapps.django_comment_common.models import ( CourseDiscussionSettings, Role, ) from openedx.core.djangoapps.django_comment_common.utils import seed_permissions_roles from openedx.core.djangoapps.oauth_dispatch.jwt import create_jwt_for_user -from openedx.core.djangoapps.oauth_dispatch.tests.factories import AccessTokenFactory, ApplicationFactory -from openedx.core.djangoapps.user_api.models import RetirementState, UserRetirementStatus +from openedx.core.djangoapps.oauth_dispatch.tests.factories import ( + AccessTokenFactory, + ApplicationFactory, +) +from openedx.core.djangoapps.user_api.models import ( + RetirementState, + UserRetirementStatus, +) +from xmodule.modulestore import ModuleStoreEnum +from xmodule.modulestore.django import modulestore +from xmodule.modulestore.tests.django_utils import ( + ModuleStoreTestCase, + SharedModuleStoreTestCase, +) +from xmodule.modulestore.tests.factories import ( + BlockFactory, + CourseFactory, + check_mongo_calls, +) -class DiscussionAPIViewTestMixin(ForumsEnableMixin, CommentsServiceMockMixin, UrlResetMixin): +class DiscussionAPIViewTestMixin( + ForumsEnableMixin, CommentsServiceMockMixin, UrlResetMixin +): """ Mixin for common code in tests of Discussion API views. This includes creation of common structures (e.g. a course, user, and enrollment), logging @@ -72,7 +100,9 @@ class DiscussionAPIViewTestMixin(ForumsEnableMixin, CommentsServiceMockMixin, Ur client_class = APIClient - @mock.patch.dict("django.conf.settings.FEATURES", {"ENABLE_DISCUSSION_SERVICE": True}) + @mock.patch.dict( + "django.conf.settings.FEATURES", {"ENABLE_DISCUSSION_SERVICE": True} + ) def setUp(self): super().setUp() self.maxDiff = None # pylint: disable=invalid-name @@ -81,7 +111,7 @@ def setUp(self): course="y", run="z", start=datetime.now(UTC), - discussion_topics={"Test Topic": {"id": "test_topic"}} + discussion_topics={"Test Topic": {"id": "test_topic"}}, ) self.password = "Password1234" self.user = UserFactory.create(password=self.password) @@ -96,23 +126,25 @@ def assert_response_correct(self, response, expected_status, expected_content): Assert that the response has the given status code and parsed content """ assert response.status_code == expected_status - parsed_content = json.loads(response.content.decode('utf-8')) + parsed_content = json.loads(response.content.decode("utf-8")) assert parsed_content == expected_content def register_thread(self, overrides=None): """ Create cs_thread with minimal fields and register response """ - cs_thread = make_minimal_cs_thread({ - "id": "test_thread", - "course_id": str(self.course.id), - "commentable_id": "test_topic", - "username": self.user.username, - "user_id": str(self.user.id), - "thread_type": "discussion", - "title": "Test Title", - "body": "Test body", - }) + cs_thread = make_minimal_cs_thread( + { + "id": "test_thread", + "course_id": str(self.course.id), + "commentable_id": "test_topic", + "username": self.user.username, + "user_id": str(self.user.id), + "thread_type": "discussion", + "title": "Test Title", + "body": "Test body", + } + ) cs_thread.update(overrides or {}) self.register_get_thread_response(cs_thread) self.register_put_thread_response(cs_thread) @@ -121,14 +153,16 @@ def register_comment(self, overrides=None): """ Create cs_comment with minimal fields and register response """ - cs_comment = make_minimal_cs_comment({ - "id": "test_comment", - "course_id": str(self.course.id), - "thread_id": "test_thread", - "username": self.user.username, - "user_id": str(self.user.id), - "body": "Original body", - }) + cs_comment = make_minimal_cs_comment( + { + "id": "test_comment", + "course_id": str(self.course.id), + "thread_id": "test_thread", + "username": self.user.username, + "user_id": str(self.user.id), + "body": "Original body", + } + ) cs_comment.update(overrides or {}) self.register_get_comment_response(cs_comment) self.register_put_comment_response(cs_comment) @@ -140,7 +174,7 @@ def test_not_authenticated(self): self.assert_response_correct( response, 401, - {"developer_message": "Authentication credentials were not provided."} + {"developer_message": "Authentication credentials were not provided."}, ) def test_inactive(self): @@ -149,12 +183,16 @@ def test_inactive(self): @mock.patch.dict("django.conf.settings.FEATURES", {"ENABLE_DISCUSSION_SERVICE": True}) -class UploadFileViewTest(ForumsEnableMixin, CommentsServiceMockMixin, UrlResetMixin, ModuleStoreTestCase): +class UploadFileViewTest( + ForumsEnableMixin, CommentsServiceMockMixin, UrlResetMixin, ModuleStoreTestCase +): """ Tests for UploadFileView. """ - @mock.patch.dict("django.conf.settings.FEATURES", {"ENABLE_DISCUSSION_SERVICE": True}) + @mock.patch.dict( + "django.conf.settings.FEATURES", {"ENABLE_DISCUSSION_SERVICE": True} + ) def setUp(self): super().setUp() self.valid_file = { @@ -165,11 +203,13 @@ def setUp(self): ), } self.user = UserFactory.create(password=self.TEST_PASSWORD) - self.course = CourseFactory.create(org='a', course='b', run='c', start=datetime.now(UTC)) + self.course = CourseFactory.create( + org="a", course="b", run="c", start=datetime.now(UTC) + ) self.url = reverse("upload_file", kwargs={"course_id": str(self.course.id)}) patcher = mock.patch( - 'openedx.core.djangoapps.discussions.config.waffle.ENABLE_FORUM_V2.is_enabled', - return_value=False + "openedx.core.djangoapps.discussions.config.waffle.ENABLE_FORUM_V2.is_enabled", + return_value=False, ) patcher.start() self.addCleanup(patcher.stop) @@ -257,10 +297,13 @@ def test_file_upload_with_thread_key(self): """ self.user_login() self.enroll_user_in_course() - response = self.client.post(self.url, { - **self.valid_file, - "thread_key": "somethread", - }) + response = self.client.post( + self.url, + { + **self.valid_file, + "thread_key": "somethread", + }, + ) response_data = json.loads(response.content) assert "/somethread/" in response_data["location"] @@ -314,7 +357,9 @@ class CommentViewSetListByUserTest( Common test cases for views retrieving user-published content. """ - @mock.patch.dict("django.conf.settings.FEATURES", {"ENABLE_DISCUSSION_SERVICE": True}) + @mock.patch.dict( + "django.conf.settings.FEATURES", {"ENABLE_DISCUSSION_SERVICE": True} + ) def setUp(self): super().setUp() @@ -323,8 +368,8 @@ def setUp(self): self.addCleanup(httpretty.reset) self.addCleanup(httpretty.disable) patcher = mock.patch( - 'openedx.core.djangoapps.discussions.config.waffle.ENABLE_FORUM_V2.is_enabled', - return_value=False + "openedx.core.djangoapps.discussions.config.waffle.ENABLE_FORUM_V2.is_enabled", + return_value=False, ) patcher.start() self.addCleanup(patcher.stop) @@ -335,7 +380,9 @@ def setUp(self): self.other_user = UserFactory.create(password=self.TEST_PASSWORD) self.register_get_user_response(self.other_user) - self.course = CourseFactory.create(org="a", course="b", run="c", start=datetime.now(UTC)) + self.course = CourseFactory.create( + org="a", course="b", run="c", start=datetime.now(UTC) + ) CourseEnrollmentFactory.create(user=self.user, course_id=self.course.id) self.url = self.build_url(self.user.username, self.course.id) @@ -346,16 +393,18 @@ def register_mock_endpoints(self): """ self.register_get_threads_response( threads=[ - make_minimal_cs_thread({ - "id": f"test_thread_{index}", - "course_id": str(self.course.id), - "commentable_id": f"test_topic_{index}", - "username": self.user.username, - "user_id": str(self.user.id), - "thread_type": "discussion", - "title": f"Test Title #{index}", - "body": f"Test body #{index}", - }) + make_minimal_cs_thread( + { + "id": f"test_thread_{index}", + "course_id": str(self.course.id), + "commentable_id": f"test_topic_{index}", + "username": self.user.username, + "user_id": str(self.user.id), + "thread_type": "discussion", + "title": f"Test Title #{index}", + "body": f"Test body #{index}", + } + ) for index in range(30) ], page=1, @@ -363,16 +412,18 @@ def register_mock_endpoints(self): ) self.register_get_comments_response( comments=[ - make_minimal_cs_comment({ - "id": f"test_comment_{index}", - "thread_id": "test_thread", - "user_id": str(self.user.id), - "username": self.user.username, - "created_at": "2015-05-11T00:00:00Z", - "updated_at": "2015-05-11T11:11:11Z", - "body": f"Test body #{index}", - "votes": {"up_count": 4}, - }) + make_minimal_cs_comment( + { + "id": f"test_comment_{index}", + "thread_id": "test_thread", + "user_id": str(self.user.id), + "username": self.user.username, + "created_at": "2015-05-11T00:00:00Z", + "updated_at": "2015-05-11T11:11:11Z", + "body": f"Test body #{index}", + "votes": {"up_count": 4}, + } + ) for index in range(30) ], page=1, @@ -384,11 +435,13 @@ def build_url(self, username, course_id, **kwargs): Builds an URL to access content from an user on a specific course. """ base = reverse("comment-list") - query = urlencode({ - "username": username, - "course_id": str(course_id), - **kwargs, - }) + query = urlencode( + { + "username": username, + "course_id": str(course_id), + **kwargs, + } + ) return f"{base}?{query}" def assert_successful_response(self, response): @@ -414,7 +467,9 @@ def test_request_by_unauthorized_user(self): they're not either enrolled or staff members. """ self.register_mock_endpoints() - self.client.login(username=self.other_user.username, password=self.TEST_PASSWORD) + self.client.login( + username=self.other_user.username, password=self.TEST_PASSWORD + ) response = self.client.get(self.url) assert response.status_code == status.HTTP_404_NOT_FOUND assert json.loads(response.content)["developer_message"] == "Course not found." @@ -425,7 +480,9 @@ def test_request_by_enrolled_user(self): comments in that course. """ self.register_mock_endpoints() - self.client.login(username=self.other_user.username, password=self.TEST_PASSWORD) + self.client.login( + username=self.other_user.username, password=self.TEST_PASSWORD + ) CourseEnrollmentFactory.create(user=self.other_user, course_id=self.course.id) self.assert_successful_response(self.client.get(self.url)) @@ -434,7 +491,9 @@ def test_request_by_global_staff(self): Staff users are allowed to get any user's comments. """ self.register_mock_endpoints() - self.client.login(username=self.other_user.username, password=self.TEST_PASSWORD) + self.client.login( + username=self.other_user.username, password=self.TEST_PASSWORD + ) GlobalStaff().add_users(self.other_user) self.assert_successful_response(self.client.get(self.url)) @@ -445,7 +504,9 @@ def test_request_by_course_staff(self, role): course. """ self.register_mock_endpoints() - self.client.login(username=self.other_user.username, password=self.TEST_PASSWORD) + self.client.login( + username=self.other_user.username, password=self.TEST_PASSWORD + ) role(course_key=self.course.id).add_users(self.other_user) self.assert_successful_response(self.client.get(self.url)) @@ -454,7 +515,9 @@ def test_request_with_non_existent_user(self): Requests for users that don't exist result in a 404 response. """ self.register_mock_endpoints() - self.client.login(username=self.other_user.username, password=self.TEST_PASSWORD) + self.client.login( + username=self.other_user.username, password=self.TEST_PASSWORD + ) GlobalStaff().add_users(self.other_user) url = self.build_url("non_existent", self.course.id) response = self.client.get(url) @@ -465,7 +528,9 @@ def test_request_with_non_existent_course(self): Requests for courses that don't exist result in a 404 response. """ self.register_mock_endpoints() - self.client.login(username=self.other_user.username, password=self.TEST_PASSWORD) + self.client.login( + username=self.other_user.username, password=self.TEST_PASSWORD + ) GlobalStaff().add_users(self.other_user) url = self.build_url(self.user.username, "course-v1:x+y+z") response = self.client.get(url) @@ -476,14 +541,18 @@ def test_request_with_invalid_course_id(self): Requests with invalid course ID should fail form validation. """ self.register_mock_endpoints() - self.client.login(username=self.other_user.username, password=self.TEST_PASSWORD) + self.client.login( + username=self.other_user.username, password=self.TEST_PASSWORD + ) GlobalStaff().add_users(self.other_user) url = self.build_url(self.user.username, "an invalid course") response = self.client.get(url) assert response.status_code == status.HTTP_400_BAD_REQUEST parsed_response = json.loads(response.content) - assert parsed_response["field_errors"]["course_id"]["developer_message"] == \ - "'an invalid course' is not a valid course id" + assert ( + parsed_response["field_errors"]["course_id"]["developer_message"] + == "'an invalid course' is not a valid course id" + ) def test_request_with_empty_results_page(self): """ @@ -493,7 +562,9 @@ def test_request_with_empty_results_page(self): self.register_get_threads_response(threads=[], page=1, num_pages=1) self.register_get_comments_response(comments=[], page=1, num_pages=1) - self.client.login(username=self.other_user.username, password=self.TEST_PASSWORD) + self.client.login( + username=self.other_user.username, password=self.TEST_PASSWORD + ) GlobalStaff().add_users(self.other_user) url = self.build_url(self.user.username, self.course.id, page=2) response = self.client.get(url) @@ -501,17 +572,23 @@ def test_request_with_empty_results_page(self): @mock.patch.dict("django.conf.settings.FEATURES", {"ENABLE_DISCUSSION_SERVICE": True}) -@override_settings(DISCUSSION_MODERATION_EDIT_REASON_CODES={"test-edit-reason": "Test Edit Reason"}) -@override_settings(DISCUSSION_MODERATION_CLOSE_REASON_CODES={"test-close-reason": "Test Close Reason"}) +@override_settings( + DISCUSSION_MODERATION_EDIT_REASON_CODES={"test-edit-reason": "Test Edit Reason"} +) +@override_settings( + DISCUSSION_MODERATION_CLOSE_REASON_CODES={"test-close-reason": "Test Close Reason"} +) class CourseViewTest(DiscussionAPIViewTestMixin, ModuleStoreTestCase): """Tests for CourseView""" def setUp(self): super().setUp() - self.url = reverse("discussion_course", kwargs={"course_id": str(self.course.id)}) + self.url = reverse( + "discussion_course", kwargs={"course_id": str(self.course.id)} + ) patcher = mock.patch( - 'openedx.core.djangoapps.discussions.config.waffle.ENABLE_FORUM_V2.is_enabled', - return_value=False + "openedx.core.djangoapps.discussions.config.waffle.ENABLE_FORUM_V2.is_enabled", + return_value=False, ) patcher.start() self.addCleanup(patcher.stop) @@ -521,9 +598,7 @@ def test_404(self): reverse("course_topics", kwargs={"course_id": "non/existent/course"}) ) self.assert_response_correct( - response, - 404, - {"developer_message": "Course not found."} + response, 404, {"developer_message": "Course not found."} ) def test_basic(self): @@ -547,23 +622,27 @@ def test_basic(self): "allow_anonymous_to_peers": False, "has_bulk_delete_privileges": False, "has_moderation_privileges": False, - 'is_course_admin': False, - 'is_course_staff': False, + "is_course_admin": False, + "is_course_staff": False, "is_group_ta": False, - 'is_user_admin': False, + "is_user_admin": False, "user_roles": ["Student"], - "edit_reasons": [{"code": "test-edit-reason", "label": "Test Edit Reason"}], - "post_close_reasons": [{"code": "test-close-reason", "label": "Test Close Reason"}], - 'show_discussions': True, - 'is_notify_all_learners_enabled': False, - 'captcha_settings': { - 'enabled': False, - 'site_key': None, + "edit_reasons": [ + {"code": "test-edit-reason", "label": "Test Edit Reason"} + ], + "post_close_reasons": [ + {"code": "test-close-reason", "label": "Test Close Reason"} + ], + "show_discussions": True, + "is_notify_all_learners_enabled": False, + "captcha_settings": { + "enabled": False, + "site_key": None, }, "is_email_verified": True, "only_verified_users_can_post": False, - "content_creation_rate_limited": False - } + "content_creation_rate_limited": False, + }, ) @@ -574,8 +653,10 @@ class RetireViewTest(DiscussionAPIViewTestMixin, ModuleStoreTestCase): def setUp(self): super().setUp() - RetirementState.objects.create(state_name='PENDING', state_execution_order=1) - self.retire_forums_state = RetirementState.objects.create(state_name='RETIRE_FORUMS', state_execution_order=11) + RetirementState.objects.create(state_name="PENDING", state_execution_order=1) + self.retire_forums_state = RetirementState.objects.create( + state_name="RETIRE_FORUMS", state_execution_order=11 + ) self.retirement = UserRetirementStatus.create_retirement(self.user) self.retirement.current_state = self.retire_forums_state @@ -586,8 +667,8 @@ def setUp(self): self.retired_username = get_retired_username_by_username(self.user.username) self.url = reverse("retire_discussion_user") patcher = mock.patch( - 'openedx.core.djangoapps.discussions.config.waffle.ENABLE_FORUM_V2.is_enabled', - return_value=False + "openedx.core.djangoapps.discussions.config.waffle.ENABLE_FORUM_V2.is_enabled", + return_value=False, ) patcher.start() self.addCleanup(patcher.stop) @@ -599,14 +680,14 @@ def assert_response_correct(self, response, expected_status, expected_content): assert response.status_code == expected_status if expected_content: - assert response.content.decode('utf-8') == expected_content + assert response.content.decode("utf-8") == expected_content def build_jwt_headers(self, user): """ Helper function for creating headers for the JWT authentication. """ token = create_jwt_for_user(user) - headers = {'HTTP_AUTHORIZATION': 'JWT ' + token} + headers = {"HTTP_AUTHORIZATION": "JWT " + token} return headers def test_basic(self): @@ -615,7 +696,7 @@ def test_basic(self): """ self.register_get_user_retire_response(self.user) headers = self.build_jwt_headers(self.superuser) - data = {'username': self.user.username} + data = {"username": self.user.username} response = self.superuser_client.post(self.url, data, **headers) self.assert_response_correct(response, 204, b"") @@ -623,9 +704,11 @@ def test_downstream_forums_error(self): """ Check that we bubble up errors from the comments service """ - self.register_get_user_retire_response(self.user, status=500, body="Server error") + self.register_get_user_retire_response( + self.user, status=500, body="Server error" + ) headers = self.build_jwt_headers(self.superuser) - data = {'username': self.user.username} + data = {"username": self.user.username} response = self.superuser_client.post(self.url, data, **headers) self.assert_response_correct(response, 500, '"Server error"') @@ -635,7 +718,7 @@ def test_nonexistent_user(self): """ nonexistent_username = "nonexistent user" self.retired_username = get_retired_username_by_username(nonexistent_username) - data = {'username': nonexistent_username} + data = {"username": nonexistent_username} headers = self.build_jwt_headers(self.superuser) response = self.superuser_client.post(self.url, data, **headers) self.assert_response_correct(response, 404, None) @@ -649,7 +732,10 @@ def test_not_authenticated(self): @ddt.ddt @httpretty.activate -@mock.patch('django.conf.settings.USERNAME_REPLACEMENT_WORKER', 'test_replace_username_service_worker') +@mock.patch( + "django.conf.settings.USERNAME_REPLACEMENT_WORKER", + "test_replace_username_service_worker", +) @mock.patch.dict("django.conf.settings.FEATURES", {"ENABLE_DISCUSSION_SERVICE": True}) class ReplaceUsernamesViewTest(DiscussionAPIViewTestMixin, ModuleStoreTestCase): """Tests for ReplaceUsernamesView""" @@ -662,8 +748,8 @@ def setUp(self): self.new_username = "test_username_replacement" self.url = reverse("replace_discussion_username") patcher = mock.patch( - 'openedx.core.djangoapps.discussions.config.waffle.ENABLE_FORUM_V2.is_enabled', - return_value=False + "openedx.core.djangoapps.discussions.config.waffle.ENABLE_FORUM_V2.is_enabled", + return_value=False, ) patcher.start() self.addCleanup(patcher.stop) @@ -682,34 +768,28 @@ def build_jwt_headers(self, user): Helper function for creating headers for the JWT authentication. """ token = create_jwt_for_user(user) - headers = {'HTTP_AUTHORIZATION': 'JWT ' + token} + headers = {"HTTP_AUTHORIZATION": "JWT " + token} return headers def call_api(self, user, client, data): - """ Helper function to call API with data """ + """Helper function to call API with data""" data = json.dumps(data) headers = self.build_jwt_headers(user) - return client.post(self.url, data, content_type='application/json', **headers) + return client.post(self.url, data, content_type="application/json", **headers) - @ddt.data( - [{}, {}], - {}, - [{"test_key": "test_value", "test_key_2": "test_value_2"}] - ) + @ddt.data([{}, {}], {}, [{"test_key": "test_value", "test_key_2": "test_value_2"}]) def test_bad_schema(self, mapping_data): - """ Verify the endpoint rejects bad data schema """ - data = { - "username_mappings": mapping_data - } + """Verify the endpoint rejects bad data schema""" + data = {"username_mappings": mapping_data} response = self.call_api(self.worker, self.worker_client, data) assert response.status_code == 400 def test_auth(self): - """ Verify the endpoint only works with the service worker """ + """Verify the endpoint only works with the service worker""" data = { "username_mappings": [ {"test_username_1": "test_new_username_1"}, - {"test_username_2": "test_new_username_2"} + {"test_username_2": "test_new_username_2"}, ] } @@ -727,15 +807,15 @@ def test_auth(self): assert response.status_code == 200 def test_basic(self): - """ Check successful replacement """ + """Check successful replacement""" data = { "username_mappings": [ {self.user.username: self.new_username}, ] } expected_response = { - 'failed_replacements': [], - 'successful_replacements': data["username_mappings"] + "failed_replacements": [], + "successful_replacements": data["username_mappings"], } self.register_get_username_replacement_response(self.user) response = self.call_api(self.worker, self.worker_client, data) @@ -751,7 +831,9 @@ def test_not_authenticated(self): @ddt.ddt @mock.patch.dict("django.conf.settings.FEATURES", {"ENABLE_DISCUSSION_SERVICE": True}) -class CourseTopicsViewTest(DiscussionAPIViewTestMixin, CommentsServiceMockMixin, ModuleStoreTestCase): +class CourseTopicsViewTest( + DiscussionAPIViewTestMixin, CommentsServiceMockMixin, ModuleStoreTestCase +): """ Tests for CourseTopicsView """ @@ -768,10 +850,12 @@ def setUp(self): "courseware-2": {"discussion": 4, "question": 5}, "courseware-3": {"discussion": 7, "question": 2}, } - self.register_get_course_commentable_counts_response(self.course.id, self.thread_counts_map) + self.register_get_course_commentable_counts_response( + self.course.id, self.thread_counts_map + ) patcher = mock.patch( - 'openedx.core.djangoapps.discussions.config.waffle.ENABLE_FORUM_V2.is_enabled', - return_value=False + "openedx.core.djangoapps.discussions.config.waffle.ENABLE_FORUM_V2.is_enabled", + return_value=False, ) patcher.start() self.addCleanup(patcher.stop) @@ -786,7 +870,7 @@ def create_course(self, blocks_count, module_store, topics): run="c", start=datetime.now(UTC), default_store=module_store, - discussion_topics=topics + discussion_topics=topics, ) CourseEnrollmentFactory.create(user=self.user, course_id=course.id) course_url = reverse("course_topics", kwargs={"course_id": str(course.id)}) @@ -794,10 +878,10 @@ def create_course(self, blocks_count, module_store, topics): for i in range(blocks_count): BlockFactory.create( parent_location=course.location, - category='discussion', - discussion_id=f'id_module_{i}', - discussion_category=f'Category {i}', - discussion_target=f'Discussion {i}', + category="discussion", + discussion_id=f"id_module_{i}", + discussion_category=f"Category {i}", + discussion_target=f"Discussion {i}", publish_item=False, ) return course_url, course.id @@ -812,7 +896,7 @@ def make_discussion_xblock(self, topic_id, category, subcategory, **kwargs): discussion_id=topic_id, discussion_category=category, discussion_target=subcategory, - **kwargs + **kwargs, ) def test_404(self): @@ -820,9 +904,7 @@ def test_404(self): reverse("course_topics", kwargs={"course_id": "non/existent/course"}) ) self.assert_response_correct( - response, - 404, - {"developer_message": "Course not found."} + response, 404, {"developer_message": "Course not found."} ) def test_basic(self): @@ -832,21 +914,30 @@ def test_basic(self): 200, { "courseware_topics": [], - "non_courseware_topics": [{ - "id": "test_topic", - "name": "Test Topic", - "children": [], - "thread_list_url": 'http://testserver/api/discussion/v1/threads/' - '?course_id=course-v1%3Ax%2By%2Bz&topic_id=test_topic', - "thread_counts": {"discussion": 0, "question": 0}, - }], - } + "non_courseware_topics": [ + { + "id": "test_topic", + "name": "Test Topic", + "children": [], + "thread_list_url": "http://testserver/api/discussion/v1/threads/" + "?course_id=course-v1%3Ax%2By%2Bz&topic_id=test_topic", + "thread_counts": {"discussion": 0, "question": 0}, + } + ], + }, ) @ddt.data( (2, ModuleStoreEnum.Type.split, 2, {"Test Topic 1": {"id": "test_topic_1"}}), - (2, ModuleStoreEnum.Type.split, 2, - {"Test Topic 1": {"id": "test_topic_1"}, "Test Topic 2": {"id": "test_topic_2"}}), + ( + 2, + ModuleStoreEnum.Type.split, + 2, + { + "Test Topic 1": {"id": "test_topic_1"}, + "Test Topic 2": {"id": "test_topic_2"}, + }, + ), (10, ModuleStoreEnum.Type.split, 2, {"Test Topic 1": {"id": "test_topic_1"}}), ) @ddt.unpack @@ -868,7 +959,7 @@ def test_discussion_topic_404(self): self.assert_response_correct( response, 404, - {"developer_message": "Discussion not found for 'invalid_topic_id'."} + {"developer_message": "Discussion not found for 'invalid_topic_id'."}, ) def test_topic_id(self): @@ -888,38 +979,41 @@ def test_topic_id(self): "non_courseware_topics": [], "courseware_topics": [ { - "children": [{ - "children": [], - "id": "topic_id_1", - "thread_list_url": "http://testserver/api/discussion/v1/threads/?" - "course_id=course-v1%3Ax%2By%2Bz&topic_id=topic_id_1", - "name": "test_target_1", - "thread_counts": {"discussion": 0, "question": 0}, - }], + "children": [ + { + "children": [], + "id": "topic_id_1", + "thread_list_url": "http://testserver/api/discussion/v1/threads/?" + "course_id=course-v1%3Ax%2By%2Bz&topic_id=topic_id_1", + "name": "test_target_1", + "thread_counts": {"discussion": 0, "question": 0}, + } + ], "id": None, "thread_list_url": "http://testserver/api/discussion/v1/threads/?" - "course_id=course-v1%3Ax%2By%2Bz&topic_id=topic_id_1", + "course_id=course-v1%3Ax%2By%2Bz&topic_id=topic_id_1", "name": "test_category_1", "thread_counts": None, }, { - "children": - [{ + "children": [ + { "children": [], "id": "topic_id_2", "thread_list_url": "http://testserver/api/discussion/v1/threads/?" - "course_id=course-v1%3Ax%2By%2Bz&topic_id=topic_id_2", + "course_id=course-v1%3Ax%2By%2Bz&topic_id=topic_id_2", "name": "test_target_2", "thread_counts": {"discussion": 0, "question": 0}, - }], + } + ], "id": None, "thread_list_url": "http://testserver/api/discussion/v1/threads/?" - "course_id=course-v1%3Ax%2By%2Bz&topic_id=topic_id_2", + "course_id=course-v1%3Ax%2By%2Bz&topic_id=topic_id_2", "name": "test_category_2", "thread_counts": None, - } - ] - } + }, + ], + }, ) @override_waffle_flag(ENABLE_NEW_STRUCTURE_DISCUSSIONS, True) @@ -930,45 +1024,46 @@ def test_new_course_structure_response(self): """ chapter = BlockFactory.create( parent_location=self.course.location, - category='chapter', + category="chapter", display_name="Week 1", start=datetime(2015, 3, 1, tzinfo=UTC), ) sequential = BlockFactory.create( parent_location=chapter.location, - category='sequential', + category="sequential", display_name="Lesson 1", start=datetime(2015, 3, 1, tzinfo=UTC), ) BlockFactory.create( parent_location=sequential.location, - category='vertical', - display_name='vertical', + category="vertical", + display_name="vertical", start=datetime(2015, 4, 1, tzinfo=UTC), ) DiscussionsConfiguration.objects.create( - context_key=self.course.id, - provider_type=Provider.OPEN_EDX + context_key=self.course.id, provider_type=Provider.OPEN_EDX ) update_discussions_settings_from_course_task(str(self.course.id)) response = json.loads(self.client.get(self.url).content.decode()) - keys = ['children', 'id', 'name', 'thread_counts', 'thread_list_url'] - assert list(response.keys()) == ['courseware_topics', 'non_courseware_topics'] - assert len(response['courseware_topics']) == 1 - courseware_keys = list(response['courseware_topics'][0].keys()) + keys = ["children", "id", "name", "thread_counts", "thread_list_url"] + assert list(response.keys()) == ["courseware_topics", "non_courseware_topics"] + assert len(response["courseware_topics"]) == 1 + courseware_keys = list(response["courseware_topics"][0].keys()) courseware_keys.sort() assert courseware_keys == keys - assert len(response['non_courseware_topics']) == 1 - non_courseware_keys = list(response['non_courseware_topics'][0].keys()) + assert len(response["non_courseware_topics"]) == 1 + non_courseware_keys = list(response["non_courseware_topics"][0].keys()) non_courseware_keys.sort() assert non_courseware_keys == keys @ddt.ddt -@mock.patch('lms.djangoapps.discussion.rest_api.api._get_course', mock.Mock()) +@mock.patch("lms.djangoapps.discussion.rest_api.api._get_course", mock.Mock()) @mock.patch.dict("django.conf.settings.FEATURES", {"ENABLE_DISCUSSION_SERVICE": True}) @override_waffle_flag(ENABLE_NEW_STRUCTURE_DISCUSSIONS, True) -class CourseTopicsViewV3Test(DiscussionAPIViewTestMixin, CommentsServiceMockMixin, ModuleStoreTestCase): +class CourseTopicsViewV3Test( + DiscussionAPIViewTestMixin, CommentsServiceMockMixin, ModuleStoreTestCase +): """ Tests for CourseTopicsViewV3 """ @@ -984,55 +1079,68 @@ def setUp(self) -> None: end=datetime(2028, 1, 1), enrollment_start=datetime(2020, 1, 1), enrollment_end=datetime(2028, 1, 1), - discussion_topics={"Course Wide Topic": { - "id": 'course-wide-topic', - "usage_key": None, - }} + discussion_topics={ + "Course Wide Topic": { + "id": "course-wide-topic", + "usage_key": None, + } + }, ) self.chapter = BlockFactory.create( parent_location=self.course.location, - category='chapter', + category="chapter", display_name="Week 1", start=datetime(2015, 3, 1, tzinfo=UTC), ) self.sequential = BlockFactory.create( parent_location=self.chapter.location, - category='sequential', + category="sequential", display_name="Lesson 1", start=datetime(2015, 3, 1, tzinfo=UTC), ) self.verticals = [ BlockFactory.create( parent_location=self.sequential.location, - category='vertical', - display_name='vertical', + category="vertical", + display_name="vertical", start=datetime(2015, 4, 1, tzinfo=UTC), ) ] course_key = self.course.id - self.config = DiscussionsConfiguration.objects.create(context_key=course_key, provider_type=Provider.OPEN_EDX) + self.config = DiscussionsConfiguration.objects.create( + context_key=course_key, provider_type=Provider.OPEN_EDX + ) topic_links = [] update_discussions_settings_from_course_task(str(course_key)) - topic_id_query = DiscussionTopicLink.objects.filter(context_key=course_key).values_list( - 'external_id', flat=True, + topic_id_query = DiscussionTopicLink.objects.filter( + context_key=course_key + ).values_list( + "external_id", + flat=True, ) - topic_ids = list(topic_id_query.order_by('ordering')) + topic_ids = list(topic_id_query.order_by("ordering")) DiscussionTopicLink.objects.bulk_create(topic_links) self.topic_stats = { - **{topic_id: dict(discussion=random.randint(0, 10), question=random.randint(0, 10)) - for topic_id in set(topic_ids)}, + **{ + topic_id: dict( + discussion=random.randint(0, 10), question=random.randint(0, 10) + ) + for topic_id in set(topic_ids) + }, topic_ids[0]: dict(discussion=0, question=0), } patcher = mock.patch( - 'lms.djangoapps.discussion.rest_api.api.get_course_commentable_counts', + "lms.djangoapps.discussion.rest_api.api.get_course_commentable_counts", mock.Mock(return_value=self.topic_stats), ) patcher.start() self.addCleanup(patcher.stop) - self.url = reverse("course_topics_v3", kwargs={"course_id": str(self.course.id)}) + self.url = reverse( + "course_topics_v3", kwargs={"course_id": str(self.course.id)} + ) patcher = mock.patch( - 'openedx.core.djangoapps.discussions.config.waffle.ENABLE_FORUM_V2.is_enabled', - return_value=False + "openedx.core.djangoapps.discussions.config.waffle.ENABLE_FORUM_V2.is_enabled", + return_value=False, ) patcher.start() self.addCleanup(patcher.stop) @@ -1041,12 +1149,23 @@ def test_basic(self): response = self.client.get(self.url) data = json.loads(response.content.decode()) expected_non_courseware_keys = [ - 'id', 'usage_key', 'name', 'thread_counts', 'enabled_in_context', - 'courseware' + "id", + "usage_key", + "name", + "thread_counts", + "enabled_in_context", + "courseware", ] expected_courseware_keys = [ - 'id', 'block_id', 'lms_web_url', 'legacy_web_url', 'student_view_url', - 'type', 'display_name', 'children', 'courseware' + "id", + "block_id", + "lms_web_url", + "legacy_web_url", + "student_view_url", + "type", + "display_name", + "children", + "courseware", ] assert response.status_code == 200 assert len(data) == 2 @@ -1054,11 +1173,11 @@ def test_basic(self): assert non_courseware_topic_keys == expected_non_courseware_keys courseware_topic_keys = list(data[1].keys()) assert courseware_topic_keys == expected_courseware_keys - expected_courseware_keys.remove('courseware') - sequential_keys = list(data[1]['children'][0].keys()) - assert sequential_keys == (expected_courseware_keys + ['thread_counts']) - expected_non_courseware_keys.remove('courseware') - vertical_keys = list(data[1]['children'][0]['children'][0].keys()) + expected_courseware_keys.remove("courseware") + sequential_keys = list(data[1]["children"][0].keys()) + assert sequential_keys == (expected_courseware_keys + ["thread_counts"]) + expected_non_courseware_keys.remove("courseware") + vertical_keys = list(data[1]["children"][0]["children"][0].keys()) assert vertical_keys == expected_non_courseware_keys @@ -1099,14 +1218,21 @@ def setUp(self): {"key": "close_reason", "value": None}, { "key": "comment_list_url", - "value": "http://testserver/api/discussion/v1/comments/?thread_id=test_thread" + "value": "http://testserver/api/discussion/v1/comments/?thread_id=test_thread", }, { "key": "editable_fields", "value": [ - 'abuse_flagged', 'anonymous', 'copy_link', 'following', 'raw_body', - 'read', 'title', 'topic_id', 'type' - ] + "abuse_flagged", + "anonymous", + "copy_link", + "following", + "raw_body", + "read", + "title", + "topic_id", + "type", + ], }, {"key": "endorsed_comment_list_url", "value": None}, {"key": "following", "value": False}, @@ -1117,32 +1243,39 @@ def setUp(self): {"key": "non_endorsed_comment_list_url", "value": None}, {"key": "preview_body", "value": "Test body"}, {"key": "raw_body", "value": "Test body"}, - {"key": "rendered_body", "value": "

    Test body

    "}, {"key": "response_count", "value": 0}, {"key": "topic_id", "value": "test_topic"}, {"key": "type", "value": "discussion"}, - {"key": "users", "value": { - self.user.username: { - "profile": { - "image": { - "has_image": False, - "image_url_full": "http://testserver/static/default_500.png", - "image_url_large": "http://testserver/static/default_120.png", - "image_url_medium": "http://testserver/static/default_50.png", - "image_url_small": "http://testserver/static/default_30.png", + { + "key": "users", + "value": { + self.user.username: { + "profile": { + "image": { + "has_image": False, + "image_url_full": "http://testserver/static/default_500.png", + "image_url_large": "http://testserver/static/default_120.png", + "image_url_medium": "http://testserver/static/default_50.png", + "image_url_small": "http://testserver/static/default_30.png", + } } } - } - }}, + }, + }, {"key": "vote_count", "value": 4}, {"key": "voted", "value": False}, - + {"key": "is_deleted", "value": None}, + {"key": "deleted_at", "value": None}, + {"key": "deleted_by", "value": None}, + {"key": "deleted_by_label", "value": None}, ] - self.url = reverse("discussion_learner_threads", kwargs={'course_id': str(self.course.id)}) + self.url = reverse( + "discussion_learner_threads", kwargs={"course_id": str(self.course.id)} + ) patcher = mock.patch( - 'openedx.core.djangoapps.discussions.config.waffle.ENABLE_FORUM_V2.is_enabled', - return_value=False + "openedx.core.djangoapps.discussions.config.waffle.ENABLE_FORUM_V2.is_enabled", + return_value=False, ) patcher.start() self.addCleanup(patcher.stop) @@ -1153,12 +1286,12 @@ def update_thread(self, thread): Value of these keys has been defined in setUp function """ for element in self.add_keys: - thread[element['key']] = element['value'] + thread[element["key"]] = element["value"] for pair in self.replace_keys: - thread[pair['to']] = thread.pop(pair['from']) + thread[pair["to"]] = thread.pop(pair["from"]) for key in self.remove_keys: thread.pop(key) - thread['comment_count'] += 1 + thread["comment_count"] += 1 return thread def test_basic(self): @@ -1170,22 +1303,26 @@ def test_basic(self): """ self.register_get_user_response(self.user) expected_cs_comments_response = { - "collection": [make_minimal_cs_thread({ - "id": "test_thread", - "course_id": str(self.course.id), - "commentable_id": "test_topic", - "user_id": str(self.user.id), - "username": self.user.username, - "created_at": "2015-04-28T00:00:00Z", - "updated_at": "2015-04-28T11:11:11Z", - "title": "Test Title", - "body": "Test body", - "votes": {"up_count": 4}, - "comments_count": 5, - "unread_comments_count": 3, - "closed_by_label": None, - "edit_by_label": None, - })], + "collection": [ + make_minimal_cs_thread( + { + "id": "test_thread", + "course_id": str(self.course.id), + "commentable_id": "test_topic", + "user_id": str(self.user.id), + "username": self.user.username, + "created_at": "2015-04-28T00:00:00Z", + "updated_at": "2015-04-28T11:11:11Z", + "title": "Test Title", + "body": "Test body", + "votes": {"up_count": 4}, + "comments_count": 5, + "unread_comments_count": 3, + "closed_by_label": None, + "edit_by_label": None, + } + ) + ], "page": 1, "num_pages": 1, } @@ -1193,14 +1330,14 @@ def test_basic(self): self.url += f"?username={self.user.username}" response = self.client.get(self.url) assert response.status_code == 200 - response_data = json.loads(response.content.decode('utf-8')) - expected_api_response = expected_cs_comments_response['collection'] + response_data = json.loads(response.content.decode("utf-8")) + expected_api_response = expected_cs_comments_response["collection"] for thread in expected_api_response: self.update_thread(thread) - assert response_data['results'] == expected_api_response - assert response_data['pagination'] == { + assert response_data["results"] == expected_api_response + assert response_data["pagination"] == { "next": None, "previous": None, "count": 1, @@ -1230,20 +1367,24 @@ def test_thread_type_by(self, thread_type): thread_type (str): Value of thread_type can be 'None', 'discussion' and 'question' """ - threads = [make_minimal_cs_thread({ - "id": "test_thread", - "course_id": str(self.course.id), - "commentable_id": "test_topic", - "user_id": str(self.user.id), - "username": self.user.username, - "created_at": "2015-04-28T00:00:00Z", - "updated_at": "2015-04-28T11:11:11Z", - "title": "Test Title", - "body": "Test body", - "votes": {"up_count": 4}, - "comments_count": 5, - "unread_comments_count": 3, - })] + threads = [ + make_minimal_cs_thread( + { + "id": "test_thread", + "course_id": str(self.course.id), + "commentable_id": "test_topic", + "user_id": str(self.user.id), + "username": self.user.username, + "created_at": "2015-04-28T00:00:00Z", + "updated_at": "2015-04-28T11:11:11Z", + "title": "Test Title", + "body": "Test body", + "votes": {"up_count": 4}, + "comments_count": 5, + "unread_comments_count": 3, + } + ) + ] expected_cs_comments_response = { "collection": threads, "page": 1, @@ -1257,23 +1398,26 @@ def test_thread_type_by(self, thread_type): "course_id": str(self.course.id), "username": self.user.username, "thread_type": thread_type, - } + }, ) assert response.status_code == 200 - self.assert_last_query_params({ - "user_id": [str(self.user.id)], - "course_id": [str(self.course.id)], - "page": ["1"], - "per_page": ["10"], - "thread_type": [thread_type], - "sort_key": ['activity'], - "count_flagged": ["False"] - }) + self.assert_last_query_params( + { + "user_id": [str(self.user.id)], + "course_id": [str(self.course.id)], + "page": ["1"], + "per_page": ["10"], + "thread_type": [thread_type], + "sort_key": ["activity"], + "count_flagged": ["False"], + "show_deleted": ["False"], + } + ) @ddt.data( ("last_activity_at", "activity"), ("comment_count", "comments"), - ("vote_count", "votes") + ("vote_count", "votes"), ) @ddt.unpack def test_order_by(self, http_query, cc_query): @@ -1284,20 +1428,24 @@ def test_order_by(self, http_query, cc_query): http_query (str): Query string sent in the http request cc_query (str): Query string used for the comments client service """ - threads = [make_minimal_cs_thread({ - "id": "test_thread", - "course_id": str(self.course.id), - "commentable_id": "test_topic", - "user_id": str(self.user.id), - "username": self.user.username, - "created_at": "2015-04-28T00:00:00Z", - "updated_at": "2015-04-28T11:11:11Z", - "title": "Test Title", - "body": "Test body", - "votes": {"up_count": 4}, - "comments_count": 5, - "unread_comments_count": 3, - })] + threads = [ + make_minimal_cs_thread( + { + "id": "test_thread", + "course_id": str(self.course.id), + "commentable_id": "test_topic", + "user_id": str(self.user.id), + "username": self.user.username, + "created_at": "2015-04-28T00:00:00Z", + "updated_at": "2015-04-28T11:11:11Z", + "title": "Test Title", + "body": "Test body", + "votes": {"up_count": 4}, + "comments_count": 5, + "unread_comments_count": 3, + } + ) + ] expected_cs_comments_response = { "collection": threads, "page": 1, @@ -1311,17 +1459,20 @@ def test_order_by(self, http_query, cc_query): "course_id": str(self.course.id), "username": self.user.username, "order_by": http_query, - } + }, ) assert response.status_code == 200 - self.assert_last_query_params({ - "user_id": [str(self.user.id)], - "course_id": [str(self.course.id)], - "page": ["1"], - "per_page": ["10"], - "sort_key": [cc_query], - "count_flagged": ["False"] - }) + self.assert_last_query_params( + { + "user_id": [str(self.user.id)], + "course_id": [str(self.course.id)], + "page": ["1"], + "per_page": ["10"], + "sort_key": [cc_query], + "count_flagged": ["False"], + "show_deleted": ["False"], + } + ) @ddt.data("flagged", "unanswered", "unread", "unresponded") def test_status_by(self, post_status): @@ -1332,20 +1483,24 @@ def test_status_by(self, post_status): post_status (str): Value of post_status can be 'flagged', 'unanswered' and 'unread' """ - threads = [make_minimal_cs_thread({ - "id": "test_thread", - "course_id": str(self.course.id), - "commentable_id": "test_topic", - "user_id": str(self.user.id), - "username": self.user.username, - "created_at": "2015-04-28T00:00:00Z", - "updated_at": "2015-04-28T11:11:11Z", - "title": "Test Title", - "body": "Test body", - "votes": {"up_count": 4}, - "comments_count": 5, - "unread_comments_count": 3, - })] + threads = [ + make_minimal_cs_thread( + { + "id": "test_thread", + "course_id": str(self.course.id), + "commentable_id": "test_topic", + "user_id": str(self.user.id), + "username": self.user.username, + "created_at": "2015-04-28T00:00:00Z", + "updated_at": "2015-04-28T11:11:11Z", + "title": "Test Title", + "body": "Test body", + "votes": {"up_count": 4}, + "comments_count": 5, + "unread_comments_count": 3, + } + ) + ] expected_cs_comments_response = { "collection": threads, "page": 1, @@ -1359,29 +1514,37 @@ def test_status_by(self, post_status): "course_id": str(self.course.id), "username": self.user.username, "status": post_status, - } + }, ) if post_status == "flagged": assert response.status_code == 403 else: assert response.status_code == 200 - self.assert_last_query_params({ - "user_id": [str(self.user.id)], - "course_id": [str(self.course.id)], - "page": ["1"], - "per_page": ["10"], - post_status: ['True'], - "sort_key": ['activity'], - "count_flagged": ["False"] - }) + self.assert_last_query_params( + { + "user_id": [str(self.user.id)], + "course_id": [str(self.course.id)], + "page": ["1"], + "per_page": ["10"], + post_status: ["True"], + "sort_key": ["activity"], + "count_flagged": ["False"], + "show_deleted": ["False"], + } + ) @ddt.ddt -class CourseDiscussionSettingsAPIViewTest(APITestCase, UrlResetMixin, ModuleStoreTestCase): +class CourseDiscussionSettingsAPIViewTest( + APITestCase, UrlResetMixin, ModuleStoreTestCase +): """ Test the course discussion settings handler API endpoint. """ - @mock.patch.dict("django.conf.settings.FEATURES", {"ENABLE_DISCUSSION_SERVICE": True}) + + @mock.patch.dict( + "django.conf.settings.FEATURES", {"ENABLE_DISCUSSION_SERVICE": True} + ) def setUp(self): super().setUp() self.course = CourseFactory.create( @@ -1389,24 +1552,26 @@ def setUp(self): course="y", run="z", start=datetime.now(UTC), - discussion_topics={"Test Topic": {"id": "test_topic"}} + discussion_topics={"Test Topic": {"id": "test_topic"}}, + ) + self.path = reverse( + "discussion_course_settings", kwargs={"course_id": str(self.course.id)} ) - self.path = reverse('discussion_course_settings', kwargs={'course_id': str(self.course.id)}) self.password = self.TEST_PASSWORD - self.user = UserFactory(username='staff', password=self.password, is_staff=True) + self.user = UserFactory(username="staff", password=self.password, is_staff=True) patcher = mock.patch( - 'openedx.core.djangoapps.discussions.config.waffle.ENABLE_FORUM_V2.is_enabled', - return_value=False + "openedx.core.djangoapps.discussions.config.waffle.ENABLE_FORUM_V2.is_enabled", + return_value=False, ) patcher.start() self.addCleanup(patcher.stop) def _get_oauth_headers(self, user): """Return the OAuth headers for testing OAuth authentication""" - access_token = AccessTokenFactory.create(user=user, application=ApplicationFactory()).token - headers = { - 'HTTP_AUTHORIZATION': 'Bearer ' + access_token - } + access_token = AccessTokenFactory.create( + user=user, application=ApplicationFactory() + ).token + headers = {"HTTP_AUTHORIZATION": "Bearer " + access_token} return headers def _login_as_staff(self): @@ -1414,24 +1579,30 @@ def _login_as_staff(self): self.client.login(username=self.user.username, password=self.password) def _login_as_discussion_staff(self): - user = UserFactory(username='abc', password='abc') - role = Role.objects.create(name='Administrator', course_id=self.course.id) + user = UserFactory(username="abc", password="abc") + role = Role.objects.create(name="Administrator", course_id=self.course.id) role.users.set([user]) - self.client.login(username=user.username, password='abc') + self.client.login(username=user.username, password="abc") def _create_divided_discussions(self): """Create some divided discussions for testing.""" - divided_inline_discussions = ['Topic A', ] - divided_course_wide_discussions = ['Topic B', ] - divided_discussions = divided_inline_discussions + divided_course_wide_discussions + divided_inline_discussions = [ + "Topic A", + ] + divided_course_wide_discussions = [ + "Topic B", + ] + divided_discussions = ( + divided_inline_discussions + divided_course_wide_discussions + ) BlockFactory.create( parent=self.course, - category='discussion', - discussion_id=topic_name_to_id(self.course, 'Topic A'), - discussion_category='Chapter', - discussion_target='Discussion', - start=datetime.now() + category="discussion", + discussion_id=topic_name_to_id(self.course, "Topic A"), + discussion_category="Chapter", + discussion_target="Discussion", + start=datetime.now(), ) discussion_topics = { "Topic B": {"id": "Topic B"}, @@ -1440,31 +1611,36 @@ def _create_divided_discussions(self): config_course_discussions( self.course, discussion_topics=discussion_topics, - divided_discussions=divided_discussions + divided_discussions=divided_discussions, ) return divided_inline_discussions, divided_course_wide_discussions def _get_expected_response(self): """Return the default expected response before any changes to the discussion settings.""" return { - 'always_divide_inline_discussions': False, - 'divided_inline_discussions': [], - 'divided_course_wide_discussions': [], - 'id': 1, - 'division_scheme': 'cohort', - 'available_division_schemes': ['cohort'], - 'reported_content_email_notifications': False, + "always_divide_inline_discussions": False, + "divided_inline_discussions": [], + "divided_course_wide_discussions": [], + "id": 1, + "division_scheme": "cohort", + "available_division_schemes": ["cohort"], + "reported_content_email_notifications": False, } def patch_request(self, data, headers=None): headers = headers if headers else {} - return self.client.patch(self.path, json.dumps(data), content_type='application/merge-patch+json', **headers) + return self.client.patch( + self.path, + json.dumps(data), + content_type="application/merge-patch+json", + **headers, + ) def _assert_current_settings(self, expected_response): """Validate the current discussion settings against the expected response.""" response = self.client.get(self.path) assert response.status_code == 200 - content = json.loads(response.content.decode('utf-8')) + content = json.loads(response.content.decode("utf-8")) assert content == expected_response def _assert_patched_settings(self, data, expected_response): @@ -1473,7 +1649,7 @@ def _assert_patched_settings(self, data, expected_response): assert response.status_code == 204 self._assert_current_settings(expected_response) - @ddt.data('get', 'patch') + @ddt.data("get", "patch") def test_authentication_required(self, method): """Test and verify that authentication is required for this endpoint.""" self.client.logout() @@ -1481,8 +1657,8 @@ def test_authentication_required(self, method): assert response.status_code == 401 @ddt.data( - {'is_staff': False, 'get_status': 403, 'put_status': 403}, - {'is_staff': True, 'get_status': 200, 'put_status': 204}, + {"is_staff": False, "get_status": 403, "put_status": 403}, + {"is_staff": True, "get_status": 200, "put_status": 204}, ) @ddt.unpack def test_oauth(self, is_staff, get_status, put_status): @@ -1495,7 +1671,7 @@ def test_oauth(self, is_staff, get_status, put_status): assert response.status_code == get_status response = self.patch_request( - {'always_divide_inline_discussions': True}, headers + {"always_divide_inline_discussions": True}, headers ) assert response.status_code == put_status @@ -1503,66 +1679,68 @@ def test_non_existent_course_id(self): """Test the response when this endpoint is passed a non-existent course id.""" self._login_as_staff() response = self.client.get( - reverse('discussion_course_settings', kwargs={ - 'course_id': 'course-v1:a+b+c' - }) + reverse( + "discussion_course_settings", kwargs={"course_id": "course-v1:a+b+c"} + ) ) assert response.status_code == 404 def test_patch_request_by_discussion_staff(self): """Test the response when patch request is sent by a user with discussions staff role.""" self._login_as_discussion_staff() - response = self.patch_request( - {'always_divide_inline_discussions': True} - ) + response = self.patch_request({"always_divide_inline_discussions": True}) assert response.status_code == 403 def test_get_request_by_discussion_staff(self): """Test the response when get request is sent by a user with discussions staff role.""" self._login_as_discussion_staff() - divided_inline_discussions, divided_course_wide_discussions = self._create_divided_discussions() + divided_inline_discussions, divided_course_wide_discussions = ( + self._create_divided_discussions() + ) response = self.client.get(self.path) assert response.status_code == 200 expected_response = self._get_expected_response() - expected_response['divided_course_wide_discussions'] = [ - topic_name_to_id(self.course, name) for name in divided_course_wide_discussions + expected_response["divided_course_wide_discussions"] = [ + topic_name_to_id(self.course, name) + for name in divided_course_wide_discussions ] - expected_response['divided_inline_discussions'] = [ + expected_response["divided_inline_discussions"] = [ topic_name_to_id(self.course, name) for name in divided_inline_discussions ] - content = json.loads(response.content.decode('utf-8')) + content = json.loads(response.content.decode("utf-8")) assert content == expected_response def test_get_request_by_non_staff_user(self): """Test the response when get request is sent by a regular user with no staff role.""" - user = UserFactory(username='abc', password='abc') - self.client.login(username=user.username, password='abc') + user = UserFactory(username="abc", password="abc") + self.client.login(username=user.username, password="abc") response = self.client.get(self.path) assert response.status_code == 403 def test_patch_request_by_non_staff_user(self): """Test the response when patch request is sent by a regular user with no staff role.""" - user = UserFactory(username='abc', password='abc') - self.client.login(username=user.username, password='abc') - response = self.patch_request( - {'always_divide_inline_discussions': True} - ) + user = UserFactory(username="abc", password="abc") + self.client.login(username=user.username, password="abc") + response = self.patch_request({"always_divide_inline_discussions": True}) assert response.status_code == 403 def test_get_settings(self): """Test the current discussion settings against the expected response.""" - divided_inline_discussions, divided_course_wide_discussions = self._create_divided_discussions() + divided_inline_discussions, divided_course_wide_discussions = ( + self._create_divided_discussions() + ) self._login_as_staff() response = self.client.get(self.path) assert response.status_code == 200 expected_response = self._get_expected_response() - expected_response['divided_course_wide_discussions'] = [ - topic_name_to_id(self.course, name) for name in divided_course_wide_discussions + expected_response["divided_course_wide_discussions"] = [ + topic_name_to_id(self.course, name) + for name in divided_course_wide_discussions ] - expected_response['divided_inline_discussions'] = [ + expected_response["divided_inline_discussions"] = [ topic_name_to_id(self.course, name) for name in divided_inline_discussions ] - content = json.loads(response.content.decode('utf-8')) + content = json.loads(response.content.decode("utf-8")) assert content == expected_response def test_available_schemes(self): @@ -1570,18 +1748,23 @@ def test_available_schemes(self): config_course_cohorts(self.course, is_cohorted=False) self._login_as_staff() expected_response = self._get_expected_response() - expected_response['available_division_schemes'] = [] + expected_response["available_division_schemes"] = [] self._assert_current_settings(expected_response) CourseModeFactory.create(course_id=self.course.id, mode_slug=CourseMode.AUDIT) - CourseModeFactory.create(course_id=self.course.id, mode_slug=CourseMode.VERIFIED) + CourseModeFactory.create( + course_id=self.course.id, mode_slug=CourseMode.VERIFIED + ) - expected_response['available_division_schemes'] = [CourseDiscussionSettings.ENROLLMENT_TRACK] + expected_response["available_division_schemes"] = [ + CourseDiscussionSettings.ENROLLMENT_TRACK + ] self._assert_current_settings(expected_response) config_course_cohorts(self.course, is_cohorted=True) - expected_response['available_division_schemes'] = [ - CourseDiscussionSettings.COHORT, CourseDiscussionSettings.ENROLLMENT_TRACK + expected_response["available_division_schemes"] = [ + CourseDiscussionSettings.COHORT, + CourseDiscussionSettings.ENROLLMENT_TRACK, ] self._assert_current_settings(expected_response) @@ -1595,11 +1778,11 @@ def test_empty_body_patch_request(self): assert response.status_code == 400 @ddt.data( - {'abc': 123}, - {'divided_course_wide_discussions': 3}, - {'divided_inline_discussions': 'a'}, - {'always_divide_inline_discussions': ['a']}, - {'division_scheme': True} + {"abc": 123}, + {"divided_course_wide_discussions": 3}, + {"divided_inline_discussions": "a"}, + {"always_divide_inline_discussions": ["a"]}, + {"division_scheme": True}, ) def test_invalid_body_parameters(self, body): """Test the response status code on sending a PATCH request with parameters having incorrect types.""" @@ -1613,31 +1796,34 @@ def test_update_always_divide_inline_discussion_settings(self): self._login_as_staff() expected_response = self._get_expected_response() self._assert_current_settings(expected_response) - expected_response['always_divide_inline_discussions'] = True + expected_response["always_divide_inline_discussions"] = True - self._assert_patched_settings({'always_divide_inline_discussions': True}, expected_response) + self._assert_patched_settings( + {"always_divide_inline_discussions": True}, expected_response + ) def test_update_course_wide_discussion_settings(self): """Test whether the 'divided_course_wide_discussions' setting is updated.""" - discussion_topics = { - 'Topic B': {'id': 'Topic B'} - } + discussion_topics = {"Topic B": {"id": "Topic B"}} config_course_cohorts(self.course, is_cohorted=True) config_course_discussions(self.course, discussion_topics=discussion_topics) expected_response = self._get_expected_response() self._login_as_staff() self._assert_current_settings(expected_response) - expected_response['divided_course_wide_discussions'] = [ + expected_response["divided_course_wide_discussions"] = [ topic_name_to_id(self.course, "Topic B") ] self._assert_patched_settings( - {'divided_course_wide_discussions': [topic_name_to_id(self.course, "Topic B")]}, - expected_response + { + "divided_course_wide_discussions": [ + topic_name_to_id(self.course, "Topic B") + ] + }, + expected_response, ) - expected_response['divided_course_wide_discussions'] = [] + expected_response["divided_course_wide_discussions"] = [] self._assert_patched_settings( - {'divided_course_wide_discussions': []}, - expected_response + {"divided_course_wide_discussions": []}, expected_response ) def test_update_inline_discussion_settings(self): @@ -1650,17 +1836,23 @@ def test_update_inline_discussion_settings(self): now = datetime.now() BlockFactory.create( parent_location=self.course.location, - category='discussion', - discussion_id='Topic_A', - discussion_category='Chapter', - discussion_target='Discussion', - start=now + category="discussion", + discussion_id="Topic_A", + discussion_category="Chapter", + discussion_target="Discussion", + start=now, + ) + expected_response["divided_inline_discussions"] = [ + "Topic_A", + ] + self._assert_patched_settings( + {"divided_inline_discussions": ["Topic_A"]}, expected_response ) - expected_response['divided_inline_discussions'] = ['Topic_A', ] - self._assert_patched_settings({'divided_inline_discussions': ['Topic_A']}, expected_response) - expected_response['divided_inline_discussions'] = [] - self._assert_patched_settings({'divided_inline_discussions': []}, expected_response) + expected_response["divided_inline_discussions"] = [] + self._assert_patched_settings( + {"divided_inline_discussions": []}, expected_response + ) def test_update_division_scheme(self): """Test whether the 'division_scheme' setting is updated.""" @@ -1668,15 +1860,17 @@ def test_update_division_scheme(self): self._login_as_staff() expected_response = self._get_expected_response() self._assert_current_settings(expected_response) - expected_response['division_scheme'] = 'none' - self._assert_patched_settings({'division_scheme': 'none'}, expected_response) + expected_response["division_scheme"] = "none" + self._assert_patched_settings({"division_scheme": "none"}, expected_response) def test_update_reported_content_email_notifications(self): """Test whether the 'reported_content_email_notifications' setting is updated.""" config_course_cohorts(self.course, is_cohorted=True) - config_course_discussions(self.course, reported_content_email_notifications=True) + config_course_discussions( + self.course, reported_content_email_notifications=True + ) expected_response = self._get_expected_response() - expected_response['reported_content_email_notifications'] = True + expected_response["reported_content_email_notifications"] = True self._login_as_staff() self._assert_current_settings(expected_response) @@ -1686,12 +1880,15 @@ class CourseDiscussionRolesAPIViewTest(APITestCase, UrlResetMixin, ModuleStoreTe """ Test the course discussion roles management endpoint. """ - @mock.patch.dict("django.conf.settings.FEATURES", {"ENABLE_DISCUSSION_SERVICE": True}) + + @mock.patch.dict( + "django.conf.settings.FEATURES", {"ENABLE_DISCUSSION_SERVICE": True} + ) def setUp(self): super().setUp() patcher = mock.patch( - 'openedx.core.djangoapps.discussions.config.waffle.ENABLE_FORUM_V2.is_enabled', - return_value=False + "openedx.core.djangoapps.discussions.config.waffle.ENABLE_FORUM_V2.is_enabled", + return_value=False, ) patcher.start() self.addCleanup(patcher.stop) @@ -1702,26 +1899,27 @@ def setUp(self): start=datetime.now(UTC), ) self.password = self.TEST_PASSWORD - self.user = UserFactory(username='staff', password=self.password, is_staff=True) - course_key = CourseKey.from_string('course-v1:x+y+z') + self.user = UserFactory(username="staff", password=self.password, is_staff=True) + course_key = CourseKey.from_string("course-v1:x+y+z") seed_permissions_roles(course_key) - @mock.patch.dict("django.conf.settings.FEATURES", {"ENABLE_DISCUSSION_SERVICE": True}) + @mock.patch.dict( + "django.conf.settings.FEATURES", {"ENABLE_DISCUSSION_SERVICE": True} + ) def path(self, course_id=None, role=None): """Return the URL path to the endpoint based on the provided arguments.""" course_id = str(self.course.id) if course_id is None else course_id - role = 'Moderator' if role is None else role + role = "Moderator" if role is None else role return reverse( - 'discussion_course_roles', - kwargs={'course_id': course_id, 'rolename': role} + "discussion_course_roles", kwargs={"course_id": course_id, "rolename": role} ) def _get_oauth_headers(self, user): """Return the OAuth headers for testing OAuth authentication.""" - access_token = AccessTokenFactory.create(user=user, application=ApplicationFactory()).token - headers = { - 'HTTP_AUTHORIZATION': 'Bearer ' + access_token - } + access_token = AccessTokenFactory.create( + user=user, application=ApplicationFactory() + ).token + headers = {"HTTP_AUTHORIZATION": "Bearer " + access_token} return headers def _login_as_staff(self): @@ -1746,9 +1944,11 @@ def _add_users_to_role(self, users, rolename): def post(self, role, user_id, action): """Make a POST request to the endpoint using the provided parameters.""" self._login_as_staff() - return self.client.post(self.path(role=role), {'user_id': user_id, 'action': action}) + return self.client.post( + self.path(role=role), {"user_id": user_id, "action": action} + ) - @ddt.data('get', 'post') + @ddt.data("get", "post") def test_authentication_required(self, method): """Test and verify that authentication is required for this endpoint.""" self.client.logout() @@ -1761,29 +1961,31 @@ def test_oauth(self): self.client.logout() response = self.client.get(self.path(), **oauth_headers) assert response.status_code == 200 - body = {'user_id': 'staff', 'action': 'allow'} - response = self.client.post(self.path(), body, format='json', **oauth_headers) + body = {"user_id": "staff", "action": "allow"} + response = self.client.post(self.path(), body, format="json", **oauth_headers) assert response.status_code == 200 @ddt.data( - {'username': 'u1', 'is_staff': False, 'expected_status': 403}, - {'username': 'u2', 'is_staff': True, 'expected_status': 200}, + {"username": "u1", "is_staff": False, "expected_status": 403}, + {"username": "u2", "is_staff": True, "expected_status": 200}, ) @ddt.unpack def test_staff_permission_required(self, username, is_staff, expected_status): """Test and verify that only users with staff permission can access this endpoint.""" - UserFactory(username=username, password='edx', is_staff=is_staff) - self.client.login(username=username, password='edx') + UserFactory(username=username, password="edx", is_staff=is_staff) + self.client.login(username=username, password="edx") response = self.client.get(self.path()) assert response.status_code == expected_status - response = self.client.post(self.path(), {'user_id': username, 'action': 'allow'}, format='json') + response = self.client.post( + self.path(), {"user_id": username, "action": "allow"}, format="json" + ) assert response.status_code == expected_status def test_non_existent_course_id(self): """Test the response when the endpoint URL contains a non-existent course id.""" self._login_as_staff() - path = self.path(course_id='course-v1:a+b+c') + path = self.path(course_id="course-v1:a+b+c") response = self.client.get(path) assert response.status_code == 404 @@ -1794,7 +1996,7 @@ def test_non_existent_course_id(self): def test_non_existent_course_role(self): """Test the response when the endpoint URL contains a non-existent role.""" self._login_as_staff() - path = self.path(role='A') + path = self.path(role="A") response = self.client.get(path) assert response.status_code == 400 @@ -1803,10 +2005,10 @@ def test_non_existent_course_role(self): assert response.status_code == 400 @ddt.data( - {'role': 'Moderator', 'count': 0}, - {'role': 'Moderator', 'count': 1}, - {'role': 'Group Moderator', 'count': 2}, - {'role': 'Community TA', 'count': 3}, + {"role": "Moderator", "count": 0}, + {"role": "Moderator", "count": 1}, + {"role": "Group Moderator", "count": 2}, + {"role": "Community TA", "count": 3}, ) @ddt.unpack def test_get_role_members(self, role, count): @@ -1820,14 +2022,14 @@ def test_get_role_members(self, role, count): assert response.status_code == 200 - content = json.loads(response.content.decode('utf-8')) - assert content['course_id'] == 'course-v1:x+y+z' - assert len(content['results']) == count - expected_fields = ('username', 'email', 'first_name', 'last_name', 'group_name') - for item in content['results']: + content = json.loads(response.content.decode("utf-8")) + assert content["course_id"] == "course-v1:x+y+z" + assert len(content["results"]) == count + expected_fields = ("username", "email", "first_name", "last_name", "group_name") + for item in content["results"]: for expected_field in expected_fields: assert expected_field in item - assert content['division_scheme'] == 'cohort' + assert content["division_scheme"] == "cohort" def test_post_missing_body(self): """Test the response with a POST request without a body.""" @@ -1836,9 +2038,9 @@ def test_post_missing_body(self): assert response.status_code == 400 @ddt.data( - {'a': 1}, - {'user_id': 'xyz', 'action': 'allow'}, - {'user_id': 'staff', 'action': 123}, + {"a": 1}, + {"user_id": "xyz", "action": "allow"}, + {"user_id": "staff", "action": 123}, ) def test_missing_or_invalid_parameters(self, body): """ @@ -1849,82 +2051,100 @@ def test_missing_or_invalid_parameters(self, body): response = self.client.post(self.path(), body) assert response.status_code == 400 - response = self.client.post(self.path(), body, format='json') + response = self.client.post(self.path(), body, format="json") assert response.status_code == 400 @ddt.data( - {'action': 'allow', 'user_in_role': False}, - {'action': 'allow', 'user_in_role': True}, - {'action': 'revoke', 'user_in_role': False}, - {'action': 'revoke', 'user_in_role': True} + {"action": "allow", "user_in_role": False}, + {"action": "allow", "user_in_role": True}, + {"action": "revoke", "user_in_role": False}, + {"action": "revoke", "user_in_role": True}, ) @ddt.unpack def test_post_update_user_role(self, action, user_in_role): """Test the response when updating the user's role""" users = self._create_and_enroll_users(count=1) user = users[0] - role = 'Moderator' + role = "Moderator" if user_in_role: self._add_users_to_role(users, role) response = self.post(role, user.username, action) assert response.status_code == 200 - content = json.loads(response.content.decode('utf-8')) - assertion = self.assertTrue if action == 'allow' else self.assertFalse - assertion(any(user.username in x['username'] for x in content['results'])) + content = json.loads(response.content.decode("utf-8")) + assertion = self.assertTrue if action == "allow" else self.assertFalse + assertion(any(user.username in x["username"] for x in content["results"])) @ddt.ddt @httpretty.activate @override_waffle_flag(ENABLE_DISCUSSIONS_MFE, True) -class CourseActivityStatsTest(ForumsEnableMixin, UrlResetMixin, CommentsServiceMockMixin, APITestCase, - SharedModuleStoreTestCase): +class CourseActivityStatsTest( + ForumsEnableMixin, + UrlResetMixin, + CommentsServiceMockMixin, + APITestCase, + SharedModuleStoreTestCase, +): """ Tests for the course stats endpoint """ - @mock.patch.dict("django.conf.settings.FEATURES", {"ENABLE_DISCUSSION_SERVICE": True}) + @mock.patch.dict( + "django.conf.settings.FEATURES", {"ENABLE_DISCUSSION_SERVICE": True} + ) def setUp(self) -> None: super().setUp() patcher = mock.patch( - 'openedx.core.djangoapps.discussions.config.waffle.ENABLE_FORUM_V2.is_enabled', - return_value=False + "openedx.core.djangoapps.discussions.config.waffle.ENABLE_FORUM_V2.is_enabled", + return_value=False, ) patcher.start() self.addCleanup(patcher.stop) self.course = CourseFactory.create() self.course_key = str(self.course.id) seed_permissions_roles(self.course.id) - self.user = UserFactory(username='user') - self.moderator = UserFactory(username='moderator') + self.user = UserFactory(username="user") + self.moderator = UserFactory(username="moderator") moderator_role = Role.objects.get(name="Moderator", course_id=self.course.id) moderator_role.users.add(self.moderator) self.stats = [ { - "active_flags": random.randint(0, 3), - "inactive_flags": random.randint(0, 2), + "threads": random.randint(0, 10), "replies": random.randint(0, 30), "responses": random.randint(0, 100), - "threads": random.randint(0, 10), - "username": f"user-{idx}" + "deleted_threads": 0, + "deleted_replies": 0, + "deleted_responses": 0, + "active_flags": random.randint(0, 3), + "inactive_flags": random.randint(0, 2), + "username": f"user-{idx}", } for idx in range(10) ] for stat in self.stats: user = UserFactory.create( - username=stat['username'], + username=stat["username"], email=f"{stat['username']}@example.com", - password=self.TEST_PASSWORD + password=self.TEST_PASSWORD, ) - CourseEnrollment.enroll(user, self.course.id, mode='audit') + CourseEnrollment.enroll(user, self.course.id, mode="audit") - CourseEnrollment.enroll(self.moderator, self.course.id, mode='audit') - self.stats_without_flags = [{**stat, "active_flags": None, "inactive_flags": None} for stat in self.stats] + CourseEnrollment.enroll(self.moderator, self.course.id, mode="audit") + self.stats_without_flags = [ + {**stat, "active_flags": None, "inactive_flags": None} + for stat in self.stats + ] self.register_course_stats_response(self.course_key, self.stats, 1, 3) - self.url = reverse("discussion_course_activity_stats", kwargs={"course_key_string": self.course_key}) + self.url = reverse( + "discussion_course_activity_stats", + kwargs={"course_key_string": self.course_key}, + ) - @mock.patch.dict("django.conf.settings.FEATURES", {"ENABLE_DISCUSSION_SERVICE": True}) + @mock.patch.dict( + "django.conf.settings.FEATURES", {"ENABLE_DISCUSSION_SERVICE": True} + ) def test_regular_user(self): """ Tests that for a regular user stats are returned without flag counts @@ -1934,7 +2154,9 @@ def test_regular_user(self): data = response.json() assert data["results"] == self.stats_without_flags - @mock.patch.dict("django.conf.settings.FEATURES", {"ENABLE_DISCUSSION_SERVICE": True}) + @mock.patch.dict( + "django.conf.settings.FEATURES", {"ENABLE_DISCUSSION_SERVICE": True} + ) def test_moderator_user(self): """ Tests that for a moderator user stats are returned with flag counts @@ -1954,7 +2176,9 @@ def test_moderator_user(self): ("user", "recency", "recency"), ) @ddt.unpack - @mock.patch.dict("django.conf.settings.FEATURES", {"ENABLE_DISCUSSION_SERVICE": True}) + @mock.patch.dict( + "django.conf.settings.FEATURES", {"ENABLE_DISCUSSION_SERVICE": True} + ) def test_sorting(self, username, ordering_requested, ordering_performed): """ Test valid sorting options and defaults @@ -1964,15 +2188,22 @@ def test_sorting(self, username, ordering_requested, ordering_performed): if ordering_requested: params = {"order_by": ordering_requested} self.client.get(self.url, params) - assert urlparse( - httpretty.last_request().path # lint-amnesty, pylint: disable=no-member - ).path == f"/api/v1/users/{self.course_key}/stats" + assert ( + urlparse( + httpretty.last_request().path # lint-amnesty, pylint: disable=no-member + ).path + == f"/api/v1/users/{self.course_key}/stats" + ) assert parse_qs( - urlparse(httpretty.last_request().path).query # lint-amnesty, pylint: disable=no-member + urlparse( + httpretty.last_request().path + ).query # lint-amnesty, pylint: disable=no-member ).get("sort_key", None) == [ordering_performed] @ddt.data("flagged", "xyz") - @mock.patch.dict("django.conf.settings.FEATURES", {"ENABLE_DISCUSSION_SERVICE": True}) + @mock.patch.dict( + "django.conf.settings.FEATURES", {"ENABLE_DISCUSSION_SERVICE": True} + ) def test_sorting_error_regular_user(self, order_by): """ Test for invalid sorting options for regular users. @@ -1982,47 +2213,60 @@ def test_sorting_error_regular_user(self, order_by): assert "order_by" in response.json()["field_errors"] @ddt.data( - ('user', 'user-0,user-1,user-2,user-3,user-4,user-5,user-6,user-7,user-8,user-9'), - ('moderator', 'moderator'), + ( + "user", + "user-0,user-1,user-2,user-3,user-4,user-5,user-6,user-7,user-8,user-9", + ), + ("moderator", "moderator"), ) @ddt.unpack - @mock.patch.dict("django.conf.settings.FEATURES", {'ENABLE_DISCUSSION_SERVICE': True}) - def test_with_username_param(self, username_search_string, comma_separated_usernames): + @mock.patch.dict( + "django.conf.settings.FEATURES", {"ENABLE_DISCUSSION_SERVICE": True} + ) + def test_with_username_param( + self, username_search_string, comma_separated_usernames + ): """ Test for endpoint with username param. """ - params = {'username': username_search_string} + params = {"username": username_search_string} self.client.login(username=self.moderator.username, password=self.TEST_PASSWORD) self.client.get(self.url, params) - assert urlparse( - httpretty.last_request().path # lint-amnesty, pylint: disable=no-member - ).path == f'/api/v1/users/{self.course_key}/stats' + assert ( + urlparse( + httpretty.last_request().path # lint-amnesty, pylint: disable=no-member + ).path + == f"/api/v1/users/{self.course_key}/stats" + ) assert parse_qs( - urlparse(httpretty.last_request().path).query # lint-amnesty, pylint: disable=no-member - ).get('usernames', [None]) == [comma_separated_usernames] + urlparse( + httpretty.last_request().path + ).query # lint-amnesty, pylint: disable=no-member + ).get("usernames", [None]) == [comma_separated_usernames] - @mock.patch.dict("django.conf.settings.FEATURES", {'ENABLE_DISCUSSION_SERVICE': True}) + @mock.patch.dict( + "django.conf.settings.FEATURES", {"ENABLE_DISCUSSION_SERVICE": True} + ) def test_with_username_param_with_no_matches(self): """ Test for endpoint with username param with no matches. """ - params = {'username': 'unknown'} + params = {"username": "unknown"} self.client.login(username=self.moderator.username, password=self.TEST_PASSWORD) response = self.client.get(self.url, params) data = response.json() - self.assertFalse(data['results']) - assert data['pagination']['count'] == 0 + self.assertFalse(data["results"]) + assert data["pagination"]["count"] == 0 - @ddt.data( - 'user-0', - 'USER-1', - 'User-2', - 'UsEr-3' + @ddt.data("user-0", "USER-1", "User-2", "UsEr-3") + @mock.patch.dict( + "django.conf.settings.FEATURES", {"ENABLE_DISCUSSION_SERVICE": True} ) - @mock.patch.dict("django.conf.settings.FEATURES", {'ENABLE_DISCUSSION_SERVICE': True}) def test_with_username_param_case(self, username_search_string): """ Test user search function is case-insensitive. """ - response = get_usernames_from_search_string(self.course_key, username_search_string, 1, 1) + response = get_usernames_from_search_string( + self.course_key, username_search_string, 1, 1 + ) assert response == (username_search_string.lower(), 1, 1) diff --git a/lms/djangoapps/discussion/rest_api/tests/test_views_v2.py b/lms/djangoapps/discussion/rest_api/tests/test_views_v2.py index 431304a9a2b5..39e48dd41ff9 100644 --- a/lms/djangoapps/discussion/rest_api/tests/test_views_v2.py +++ b/lms/djangoapps/discussion/rest_api/tests/test_views_v2.py @@ -14,8 +14,6 @@ from unittest import mock import ddt -from forum.backends.mongodb.comments import Comment -from forum.backends.mongodb.threads import CommentThread import httpretty from django.urls import reverse from pytz import UTC @@ -23,30 +21,39 @@ from rest_framework.parsers import JSONParser from rest_framework.test import APIClient -from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase -from xmodule.modulestore.tests.factories import CourseFactory from common.djangoapps.student.tests.factories import ( CourseEnrollmentFactory, UserFactory, ) from common.djangoapps.util.testing import PatchMediaTypeMixin, UrlResetMixin from common.test.utils import disable_signal -from lms.djangoapps.discussion.tests.utils import ( - make_minimal_cs_comment, - make_minimal_cs_thread, +from forum.backends.mongodb.comments import Comment +from forum.backends.mongodb.threads import CommentThread +from lms.djangoapps.discussion.django_comment_client.tests.utils import ( + ForumsEnableMixin, ) -from lms.djangoapps.discussion.django_comment_client.tests.utils import ForumsEnableMixin from lms.djangoapps.discussion.rest_api import api from lms.djangoapps.discussion.rest_api.tests.utils import ( ForumMockUtilsMixin, ProfileImageTestMixin, make_paginated_api_response, ) +from lms.djangoapps.discussion.tests.utils import ( + make_minimal_cs_comment, + make_minimal_cs_thread, +) from openedx.core.djangoapps.django_comment_common.models import ( - FORUM_ROLE_ADMINISTRATOR, FORUM_ROLE_COMMUNITY_TA, FORUM_ROLE_MODERATOR, FORUM_ROLE_STUDENT, - assign_role + FORUM_ROLE_ADMINISTRATOR, + FORUM_ROLE_COMMUNITY_TA, + FORUM_ROLE_MODERATOR, + FORUM_ROLE_STUDENT, + assign_role, ) -from openedx.core.djangoapps.user_api.accounts.image_helpers import get_profile_image_storage +from openedx.core.djangoapps.user_api.accounts.image_helpers import ( + get_profile_image_storage, +) +from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase +from xmodule.modulestore.tests.factories import CourseFactory class DiscussionAPIViewTestMixin(ForumsEnableMixin, ForumMockUtilsMixin, UrlResetMixin): @@ -387,6 +394,10 @@ def expected_response_data(self, overrides=None): "image_url_small": "http://testserver/static/default_30.png", }, "learner_status": "new", + "is_deleted": None, + "deleted_at": None, + "deleted_by": None, + "deleted_by_label": None, } response_data.update(overrides or {}) return response_data @@ -512,15 +523,17 @@ def test_course_id_missing(self): self.assert_response_correct( response, 400, - {"field_errors": {"course_id": {"developer_message": "This field is required."}}} + { + "field_errors": { + "course_id": {"developer_message": "This field is required."} + } + }, ) def test_404(self): response = self.client.get(self.url, {"course_id": "non/existent/course"}) self.assert_response_correct( - response, - 404, - {"developer_message": "Course not found."} + response, 404, {"developer_message": "Course not found."} ) def test_basic(self): @@ -871,7 +884,9 @@ class BulkDeleteUserPostsTest(DiscussionAPIViewTestMixin, ModuleStoreTestCase): def setUp(self): super().setUp() - self.url = reverse("bulk_delete_user_posts", kwargs={"course_id": str(self.course.id)}) + self.url = reverse( + "bulk_delete_user_posts", kwargs={"course_id": str(self.course.id)} + ) self.user2 = UserFactory.create(password=self.password) CourseEnrollmentFactory.create(user=self.user2, course_id=self.course.id) @@ -887,13 +902,19 @@ def mock_comment_and_thread_count(self, comment_count=1, thread_count=1): thread_collection = mock.MagicMock() thread_collection.count_documents.return_value = thread_count patch_thread = mock.patch.object( - CommentThread, "_collection", new_callable=mock.PropertyMock, return_value=thread_collection + CommentThread, + "_collection", + new_callable=mock.PropertyMock, + return_value=thread_collection, ) comment_collection = mock.MagicMock() comment_collection.count_documents.return_value = comment_count patch_comment = mock.patch.object( - Comment, "_collection", new_callable=mock.PropertyMock, return_value=comment_collection + Comment, + "_collection", + new_callable=mock.PropertyMock, + return_value=comment_collection, ) thread_mock = patch_thread.start() @@ -908,7 +929,9 @@ def test_bulk_delete_denied_for_discussion_roles(self, role): """ Test bulk delete user posts denied with discussion roles. """ - thread_mock, comment_mock = self.mock_comment_and_thread_count(comment_count=1, thread_count=1) + thread_mock, comment_mock = self.mock_comment_and_thread_count( + comment_count=1, thread_count=1 + ) assign_role(self.course.id, self.user, role) response = self.client.post( f"{self.url}?username={self.user2.username}", @@ -932,7 +955,9 @@ def test_bulk_delete_allowed_for_discussion_roles(self, role): assert response.status_code == status.HTTP_202_ACCEPTED assert response.json() == {"comment_count": 1, "thread_count": 1} - @mock.patch('lms.djangoapps.discussion.rest_api.views.delete_course_post_for_user.apply_async') + @mock.patch( + "lms.djangoapps.discussion.rest_api.views.delete_course_post_for_user.apply_async" + ) @ddt.data(True, False) def test_task_only_runs_if_execute_param_is_true(self, execute, task_mock): """ diff --git a/lms/djangoapps/discussion/rest_api/tests/utils.py b/lms/djangoapps/discussion/rest_api/tests/utils.py index 8c1615690ad5..990e68a30af9 100644 --- a/lms/djangoapps/discussion/rest_api/tests/utils.py +++ b/lms/djangoapps/discussion/rest_api/tests/utils.py @@ -2,7 +2,6 @@ Discussion API test utilities """ - import hashlib import json import re @@ -14,11 +13,18 @@ from PIL import Image from pytz import UTC -from lms.djangoapps.discussion.django_comment_client.tests.mixins import MockForumApiMixin -from openedx.core.djangoapps.django_comment_common.comment_client.utils import CommentClientRequestError +from lms.djangoapps.discussion.django_comment_client.tests.mixins import ( + MockForumApiMixin, +) +from openedx.core.djangoapps.django_comment_common.comment_client.utils import ( + CommentClientRequestError, +) from openedx.core.djangoapps.profile_images.images import create_profile_images from openedx.core.djangoapps.profile_images.tests.helpers import make_image_file -from openedx.core.djangoapps.user_api.accounts.image_helpers import get_profile_image_names, set_has_profile_image +from openedx.core.djangoapps.user_api.accounts.image_helpers import ( + get_profile_image_names, + set_has_profile_image, +) def _get_thread_callback(thread_data): @@ -26,6 +32,7 @@ def _get_thread_callback(thread_data): Get a callback function that will return POST/PUT data overridden by response_overrides. """ + def callback(request, _uri, headers): """ Simulate the thread creation or update endpoint by returning the provided @@ -42,7 +49,7 @@ def callback(request, _uri, headers): response_data["edit_history"] = [ { "original_body": original_data["body"], - "author": thread_data.get('username'), + "author": thread_data.get("username"), "reason_code": val, }, ] @@ -68,11 +75,13 @@ def callback(*args, **kwargs): if key in ["anonymous", "anonymous_to_peers", "closed", "pinned"]: response_data[key] = val is True or val == "True" elif key == "edit_reason_code": - response_data["edit_history"] = [{ - "original_body": original_data["body"], - "author": thread_data.get("username"), - "reason_code": val, - }] + response_data["edit_history"] = [ + { + "original_body": original_data["body"], + "author": thread_data.get("username"), + "reason_code": val, + } + ] else: response_data[key] = val @@ -87,6 +96,7 @@ def _get_comment_callback(comment_data, thread_id, parent_id): plus necessary dummy data, overridden by the content of the POST/PUT request. """ + def callback(request, _uri, headers): """ Simulate the comment creation or update endpoint as described above. @@ -105,7 +115,7 @@ def callback(request, _uri, headers): response_data["edit_history"] = [ { "original_body": original_data["body"], - "author": comment_data.get('username'), + "author": comment_data.get("username"), "reason_code": val, }, ] @@ -135,11 +145,13 @@ def callback(*args, **kwargs): if key in ["anonymous", "anonymous_to_peers", "endorsed"]: response_data[key] = val is True or val == "True" elif key == "edit_reason_code": - response_data["edit_history"] = [{ - "original_body": original_data["body"], - "author": comment_data.get("username"), - "reason_code": val, - }] + response_data["edit_history"] = [ + { + "original_body": original_data["body"], + "author": comment_data.get("username"), + "reason_code": val, + } + ] else: response_data[key] = val @@ -152,9 +164,11 @@ def make_user_callbacks(user_map): """ Returns a callable that mimics user creation. """ + def callback(*args, **kwargs): - user_id = args[0] if args else kwargs.get('user_id') + user_id = args[0] if args else kwargs.get("user_id") return user_map[str(user_id)] + return callback @@ -163,54 +177,58 @@ class CommentsServiceMockMixin: def register_get_threads_response(self, threads, page, num_pages): """Register a mock response for GET on the CS thread list endpoint""" - assert httpretty.is_enabled(), 'httpretty must be enabled to mock calls.' + assert httpretty.is_enabled(), "httpretty must be enabled to mock calls." httpretty.register_uri( httpretty.GET, "http://localhost:4567/api/v1/threads", - body=json.dumps({ - "collection": threads, - "page": page, - "num_pages": num_pages, - "thread_count": len(threads), - }), - status=200 + body=json.dumps( + { + "collection": threads, + "page": page, + "num_pages": num_pages, + "thread_count": len(threads), + } + ), + status=200, ) def register_get_course_commentable_counts_response(self, course_id, thread_counts): """Register a mock response for GET on the CS thread list endpoint""" - assert httpretty.is_enabled(), 'httpretty must be enabled to mock calls.' + assert httpretty.is_enabled(), "httpretty must be enabled to mock calls." httpretty.register_uri( httpretty.GET, f"http://localhost:4567/api/v1/commentables/{course_id}/counts", body=json.dumps(thread_counts), - status=200 + status=200, ) def register_get_threads_search_response(self, threads, rewrite, num_pages=1): """Register a mock response for GET on the CS thread search endpoint""" - assert httpretty.is_enabled(), 'httpretty must be enabled to mock calls.' + assert httpretty.is_enabled(), "httpretty must be enabled to mock calls." httpretty.register_uri( httpretty.GET, "http://localhost:4567/api/v1/search/threads", - body=json.dumps({ - "collection": threads, - "page": 1, - "num_pages": num_pages, - "corrected_text": rewrite, - "thread_count": len(threads), - }), - status=200 + body=json.dumps( + { + "collection": threads, + "page": 1, + "num_pages": num_pages, + "corrected_text": rewrite, + "thread_count": len(threads), + } + ), + status=200, ) def register_post_thread_response(self, thread_data): """Register a mock response for POST on the CS commentable endpoint""" - assert httpretty.is_enabled(), 'httpretty must be enabled to mock calls.' + assert httpretty.is_enabled(), "httpretty must be enabled to mock calls." httpretty.register_uri( httpretty.POST, re.compile(r"http://localhost:4567/api/v1/(\w+)/threads"), - body=_get_thread_callback(thread_data) + body=_get_thread_callback(thread_data), ) def register_put_thread_response(self, thread_data): @@ -218,49 +236,51 @@ def register_put_thread_response(self, thread_data): Register a mock response for PUT on the CS endpoint for the given thread_id. """ - assert httpretty.is_enabled(), 'httpretty must be enabled to mock calls.' + assert httpretty.is_enabled(), "httpretty must be enabled to mock calls." httpretty.register_uri( httpretty.PUT, "http://localhost:4567/api/v1/threads/{}".format(thread_data["id"]), - body=_get_thread_callback(thread_data) + body=_get_thread_callback(thread_data), ) def register_get_thread_error_response(self, thread_id, status_code): """Register a mock error response for GET on the CS thread endpoint.""" - assert httpretty.is_enabled(), 'httpretty must be enabled to mock calls.' + assert httpretty.is_enabled(), "httpretty must be enabled to mock calls." httpretty.register_uri( httpretty.GET, f"http://localhost:4567/api/v1/threads/{thread_id}", body="", - status=status_code + status=status_code, ) def register_get_thread_response(self, thread): """ Register a mock response for GET on the CS thread instance endpoint. """ - assert httpretty.is_enabled(), 'httpretty must be enabled to mock calls.' + assert httpretty.is_enabled(), "httpretty must be enabled to mock calls." httpretty.register_uri( httpretty.GET, "http://localhost:4567/api/v1/threads/{id}".format(id=thread["id"]), body=json.dumps(thread), - status=200 + status=200, ) def register_get_comments_response(self, comments, page, num_pages): """Register a mock response for GET on the CS comments list endpoint""" - assert httpretty.is_enabled(), 'httpretty must be enabled to mock calls.' + assert httpretty.is_enabled(), "httpretty must be enabled to mock calls." httpretty.register_uri( httpretty.GET, "http://localhost:4567/api/v1/comments", - body=json.dumps({ - "collection": comments, - "page": page, - "num_pages": num_pages, - "comment_count": len(comments), - }), - status=200 + body=json.dumps( + { + "collection": comments, + "page": page, + "num_pages": num_pages, + "comment_count": len(comments), + } + ), + status=200, ) def register_post_comment_response(self, comment_data, thread_id, parent_id=None): @@ -274,11 +294,11 @@ def register_post_comment_response(self, comment_data, thread_id, parent_id=None else: url = f"http://localhost:4567/api/v1/threads/{thread_id}/comments" - assert httpretty.is_enabled(), 'httpretty must be enabled to mock calls.' + assert httpretty.is_enabled(), "httpretty must be enabled to mock calls." httpretty.register_uri( httpretty.POST, url, - body=_get_comment_callback(comment_data, thread_id, parent_id) + body=_get_comment_callback(comment_data, thread_id, parent_id), ) def register_put_comment_response(self, comment_data): @@ -288,11 +308,11 @@ def register_put_comment_response(self, comment_data): """ thread_id = comment_data["thread_id"] parent_id = comment_data.get("parent_id") - assert httpretty.is_enabled(), 'httpretty must be enabled to mock calls.' + assert httpretty.is_enabled(), "httpretty must be enabled to mock calls." httpretty.register_uri( httpretty.PUT, "http://localhost:4567/api/v1/comments/{}".format(comment_data["id"]), - body=_get_comment_callback(comment_data, thread_id, parent_id) + body=_get_comment_callback(comment_data, thread_id, parent_id), ) def register_get_comment_error_response(self, comment_id, status_code): @@ -300,12 +320,12 @@ def register_get_comment_error_response(self, comment_id, status_code): Register a mock error response for GET on the CS comment instance endpoint. """ - assert httpretty.is_enabled(), 'httpretty must be enabled to mock calls.' + assert httpretty.is_enabled(), "httpretty must be enabled to mock calls." httpretty.register_uri( httpretty.GET, f"http://localhost:4567/api/v1/comments/{comment_id}", body="", - status=status_code + status=status_code, ) def register_get_comment_response(self, response_overrides): @@ -313,75 +333,83 @@ def register_get_comment_response(self, response_overrides): Register a mock response for GET on the CS comment instance endpoint. """ comment = make_minimal_cs_comment(response_overrides) - assert httpretty.is_enabled(), 'httpretty must be enabled to mock calls.' + assert httpretty.is_enabled(), "httpretty must be enabled to mock calls." httpretty.register_uri( httpretty.GET, "http://localhost:4567/api/v1/comments/{id}".format(id=comment["id"]), body=json.dumps(comment), - status=200 + status=200, ) - def register_get_user_response(self, user, subscribed_thread_ids=None, upvoted_ids=None): + def register_get_user_response( + self, user, subscribed_thread_ids=None, upvoted_ids=None + ): """Register a mock response for GET on the CS user instance endpoint""" - assert httpretty.is_enabled(), 'httpretty must be enabled to mock calls.' + assert httpretty.is_enabled(), "httpretty must be enabled to mock calls." httpretty.register_uri( httpretty.GET, f"http://localhost:4567/api/v1/users/{user.id}", - body=json.dumps({ - "id": str(user.id), - "subscribed_thread_ids": subscribed_thread_ids or [], - "upvoted_ids": upvoted_ids or [], - }), - status=200 + body=json.dumps( + { + "id": str(user.id), + "subscribed_thread_ids": subscribed_thread_ids or [], + "upvoted_ids": upvoted_ids or [], + } + ), + status=200, ) def register_get_user_retire_response(self, user, status=200, body=""): """Register a mock response for GET on the CS user retirement endpoint""" - assert httpretty.is_enabled(), 'httpretty must be enabled to mock calls.' + assert httpretty.is_enabled(), "httpretty must be enabled to mock calls." httpretty.register_uri( httpretty.POST, f"http://localhost:4567/api/v1/users/{user.id}/retire", body=body, - status=status + status=status, ) def register_get_username_replacement_response(self, user, status=200, body=""): - assert httpretty.is_enabled(), 'httpretty must be enabled to mock calls.' + assert httpretty.is_enabled(), "httpretty must be enabled to mock calls." httpretty.register_uri( httpretty.POST, f"http://localhost:4567/api/v1/users/{user.id}/replace_username", body=body, - status=status + status=status, ) def register_subscribed_threads_response(self, user, threads, page, num_pages): """Register a mock response for GET on the CS user instance endpoint""" - assert httpretty.is_enabled(), 'httpretty must be enabled to mock calls.' + assert httpretty.is_enabled(), "httpretty must be enabled to mock calls." httpretty.register_uri( httpretty.GET, f"http://localhost:4567/api/v1/users/{user.id}/subscribed_threads", - body=json.dumps({ - "collection": threads, - "page": page, - "num_pages": num_pages, - "thread_count": len(threads), - }), - status=200 + body=json.dumps( + { + "collection": threads, + "page": page, + "num_pages": num_pages, + "thread_count": len(threads), + } + ), + status=200, ) def register_course_stats_response(self, course_key, stats, page, num_pages): """Register a mock response for GET on the CS user course stats instance endpoint""" - assert httpretty.is_enabled(), 'httpretty must be enabled to mock calls.' + assert httpretty.is_enabled(), "httpretty must be enabled to mock calls." httpretty.register_uri( httpretty.GET, f"http://localhost:4567/api/v1/users/{course_key}/stats", - body=json.dumps({ - "user_stats": stats, - "page": page, - "num_pages": num_pages, - "count": len(stats), - }), - status=200 + body=json.dumps( + { + "user_stats": stats, + "page": page, + "num_pages": num_pages, + "count": len(stats), + } + ), + status=200, ) def register_subscription_response(self, user): @@ -389,13 +417,13 @@ def register_subscription_response(self, user): Register a mock response for POST and DELETE on the CS user subscription endpoint """ - assert httpretty.is_enabled(), 'httpretty must be enabled to mock calls.' + assert httpretty.is_enabled(), "httpretty must be enabled to mock calls." for method in [httpretty.POST, httpretty.DELETE]: httpretty.register_uri( method, f"http://localhost:4567/api/v1/users/{user.id}/subscriptions", body=json.dumps({}), # body is unused - status=200 + status=200, ) def register_thread_votes_response(self, thread_id): @@ -403,13 +431,13 @@ def register_thread_votes_response(self, thread_id): Register a mock response for PUT and DELETE on the CS thread votes endpoint """ - assert httpretty.is_enabled(), 'httpretty must be enabled to mock calls.' + assert httpretty.is_enabled(), "httpretty must be enabled to mock calls." for method in [httpretty.PUT, httpretty.DELETE]: httpretty.register_uri( method, f"http://localhost:4567/api/v1/threads/{thread_id}/votes", body=json.dumps({}), # body is unused - status=200 + status=200, ) def register_comment_votes_response(self, comment_id): @@ -417,41 +445,39 @@ def register_comment_votes_response(self, comment_id): Register a mock response for PUT and DELETE on the CS comment votes endpoint """ - assert httpretty.is_enabled(), 'httpretty must be enabled to mock calls.' + assert httpretty.is_enabled(), "httpretty must be enabled to mock calls." for method in [httpretty.PUT, httpretty.DELETE]: httpretty.register_uri( method, f"http://localhost:4567/api/v1/comments/{comment_id}/votes", body=json.dumps({}), # body is unused - status=200 + status=200, ) def register_flag_response(self, content_type, content_id): """Register a mock response for PUT on the CS flag endpoints""" - assert httpretty.is_enabled(), 'httpretty must be enabled to mock calls.' + assert httpretty.is_enabled(), "httpretty must be enabled to mock calls." for path in ["abuse_flag", "abuse_unflag"]: httpretty.register_uri( "PUT", "http://localhost:4567/api/v1/{content_type}s/{content_id}/{path}".format( - content_type=content_type, - content_id=content_id, - path=path + content_type=content_type, content_id=content_id, path=path ), body=json.dumps({}), # body is unused - status=200 + status=200, ) def register_read_response(self, user, content_type, content_id): """ Register a mock response for POST on the CS 'read' endpoint """ - assert httpretty.is_enabled(), 'httpretty must be enabled to mock calls.' + assert httpretty.is_enabled(), "httpretty must be enabled to mock calls." httpretty.register_uri( httpretty.POST, f"http://localhost:4567/api/v1/users/{user.id}/read", - params={'source_type': content_type, 'source_id': content_id}, + params={"source_type": content_type, "source_id": content_id}, body=json.dumps({}), # body is unused - status=200 + status=200, ) def register_thread_flag_response(self, thread_id): @@ -466,48 +492,48 @@ def register_delete_thread_response(self, thread_id): """ Register a mock response for DELETE on the CS thread instance endpoint """ - assert httpretty.is_enabled(), 'httpretty must be enabled to mock calls.' + assert httpretty.is_enabled(), "httpretty must be enabled to mock calls." httpretty.register_uri( httpretty.DELETE, f"http://localhost:4567/api/v1/threads/{thread_id}", body=json.dumps({}), # body is unused - status=200 + status=200, ) def register_delete_comment_response(self, comment_id): """ Register a mock response for DELETE on the CS comment instance endpoint """ - assert httpretty.is_enabled(), 'httpretty must be enabled to mock calls.' + assert httpretty.is_enabled(), "httpretty must be enabled to mock calls." httpretty.register_uri( httpretty.DELETE, f"http://localhost:4567/api/v1/comments/{comment_id}", body=json.dumps({}), # body is unused - status=200 + status=200, ) def register_user_active_threads(self, user_id, response): """ Register a mock response for GET on the CS comment active threads endpoint """ - assert httpretty.is_enabled(), 'httpretty must be enabled to mock calls.' + assert httpretty.is_enabled(), "httpretty must be enabled to mock calls." httpretty.register_uri( httpretty.GET, f"http://localhost:4567/api/v1/users/{user_id}/active_threads", body=json.dumps(response), - status=200 + status=200, ) def register_get_subscriptions(self, thread_id, response): """ Register a mock response for GET on the CS comment active threads endpoint """ - assert httpretty.is_enabled(), 'httpretty must be enabled to mock calls.' + assert httpretty.is_enabled(), "httpretty must be enabled to mock calls." httpretty.register_uri( httpretty.GET, f"http://localhost:4567/api/v1/threads/{thread_id}/subscriptions", body=json.dumps(response), - status=200 + status=200, ) def assert_query_params_equal(self, httpretty_request, expected_params): @@ -531,7 +557,7 @@ def request_patch(self, request_data): return self.client.patch( self.url, json.dumps(request_data), - content_type="application/merge-patch+json" + content_type="application/merge-patch+json", ) def expected_thread_data(self, overrides=None): @@ -589,6 +615,10 @@ def expected_thread_data(self, overrides=None): "close_reason": None, "close_reason_code": None, "learner_status": "new", + "is_deleted": None, + "deleted_at": None, + "deleted_by": None, + "deleted_by_label": None, } response_data.update(overrides or {}) return response_data @@ -599,137 +629,153 @@ class ForumMockUtilsMixin(MockForumApiMixin): def register_get_threads_response(self, threads, page, num_pages): """Register a mock response for GET on the CS thread list endpoint""" - self.set_mock_return_value('get_user_threads', { - "collection": threads, - "page": page, - "num_pages": num_pages, - "thread_count": len(threads), - }) + self.set_mock_return_value( + "get_user_threads", + { + "collection": threads, + "page": page, + "num_pages": num_pages, + "thread_count": len(threads), + }, + ) def register_get_course_commentable_counts_response(self, course_id, thread_counts): """Register a mock response for GET on the CS thread list endpoint""" - self.set_mock_return_value('get_commentables_stats', thread_counts) + self.set_mock_return_value("get_commentables_stats", thread_counts) def register_get_threads_search_response(self, threads, rewrite, num_pages=1): """Register a mock response for GET on the CS thread search endpoint""" - self.set_mock_return_value('search_threads', { - "collection": threads, - "page": 1, - "num_pages": num_pages, - "corrected_text": rewrite, - "thread_count": len(threads), - }) + self.set_mock_return_value( + "search_threads", + { + "collection": threads, + "page": 1, + "num_pages": num_pages, + "corrected_text": rewrite, + "thread_count": len(threads), + }, + ) def register_post_thread_response(self, thread_data): - self.set_mock_side_effect('create_thread', make_thread_callback(thread_data)) + self.set_mock_side_effect("create_thread", make_thread_callback(thread_data)) def register_put_thread_response(self, thread_data): - self.set_mock_side_effect('update_thread', make_thread_callback(thread_data)) + self.set_mock_side_effect("update_thread", make_thread_callback(thread_data)) def register_get_thread_error_response(self, thread_id, status_code): self.set_mock_side_effect( - 'get_thread', - CommentClientRequestError(f"Thread does not exist with Id: {thread_id}") + "get_thread", + CommentClientRequestError(f"Thread does not exist with Id: {thread_id}"), ) def register_get_thread_response(self, thread): - self.set_mock_return_value('get_thread', thread) + self.set_mock_return_value("get_thread", thread) def register_get_comments_response(self, comments, page, num_pages): - self.set_mock_return_value('get_parent_comment', { - "collection": comments, - "page": page, - "num_pages": num_pages, - "comment_count": len(comments), - }) + self.set_mock_return_value( + "get_parent_comment", + { + "collection": comments, + "page": page, + "num_pages": num_pages, + "comment_count": len(comments), + }, + ) def register_post_comment_response(self, comment_data, thread_id, parent_id=None): self.set_mock_side_effect( - 'create_child_comment' if parent_id else 'create_parent_comment', - make_comment_callback(comment_data, thread_id, parent_id) + "create_child_comment" if parent_id else "create_parent_comment", + make_comment_callback(comment_data, thread_id, parent_id), ) def register_put_comment_response(self, comment_data): thread_id = comment_data["thread_id"] parent_id = comment_data.get("parent_id") self.set_mock_side_effect( - 'update_comment', - make_comment_callback(comment_data, thread_id, parent_id) + "update_comment", make_comment_callback(comment_data, thread_id, parent_id) ) def register_get_comment_error_response(self, comment_id, status_code): self.set_mock_side_effect( - 'get_parent_comment', - CommentClientRequestError(f"Comment does not exist with Id: {comment_id}") + "get_parent_comment", + CommentClientRequestError(f"Comment does not exist with Id: {comment_id}"), ) def register_get_comment_response(self, response_overrides): comment = make_minimal_cs_comment(response_overrides) - self.set_mock_return_value('get_parent_comment', comment) + self.set_mock_return_value("get_parent_comment", comment) - def register_get_user_response(self, user, subscribed_thread_ids=None, upvoted_ids=None): + def register_get_user_response( + self, user, subscribed_thread_ids=None, upvoted_ids=None + ): """Register a mock response for GET on the CS user endpoint""" self.users_map[str(user.id)] = { "id": str(user.id), "subscribed_thread_ids": subscribed_thread_ids or [], "upvoted_ids": upvoted_ids or [], } - self.set_mock_side_effect('get_user', make_user_callbacks(self.users_map)) + self.set_mock_side_effect("get_user", make_user_callbacks(self.users_map)) def register_get_user_retire_response(self, user, body=""): - self.set_mock_return_value('retire_user', body) + self.set_mock_return_value("retire_user", body) def register_get_username_replacement_response(self, user, status=200, body=""): - self.set_mock_return_value('update_username', body) + self.set_mock_return_value("update_username", body) def register_subscribed_threads_response(self, user, threads, page, num_pages): - self.set_mock_return_value('get_user_subscriptions', { - "collection": threads, - "page": page, - "num_pages": num_pages, - "thread_count": len(threads), - }) + self.set_mock_return_value( + "get_user_subscriptions", + { + "collection": threads, + "page": page, + "num_pages": num_pages, + "thread_count": len(threads), + }, + ) def register_course_stats_response(self, course_key, stats, page, num_pages): - self.set_mock_return_value('get_user_course_stats', { - "user_stats": stats, - "page": page, - "num_pages": num_pages, - "count": len(stats), - }) + self.set_mock_return_value( + "get_user_course_stats", + { + "user_stats": stats, + "page": page, + "num_pages": num_pages, + "count": len(stats), + }, + ) def register_subscription_response(self, user): - self.set_mock_return_value('create_subscription', {}) - self.set_mock_return_value('delete_subscription', {}) + self.set_mock_return_value("create_subscription", {}) + self.set_mock_return_value("delete_subscription", {}) def register_thread_votes_response(self, thread_id): - self.set_mock_return_value('update_thread_votes', {}) - self.set_mock_return_value('delete_thread_vote', {}) + self.set_mock_return_value("update_thread_votes", {}) + self.set_mock_return_value("delete_thread_vote", {}) def register_comment_votes_response(self, comment_id): - self.set_mock_return_value('update_comment_votes', {}) - self.set_mock_return_value('delete_comment_vote', {}) + self.set_mock_return_value("update_comment_votes", {}) + self.set_mock_return_value("delete_comment_vote", {}) def register_flag_response(self, content_type, content_id): - if content_type == 'thread': - self.set_mock_return_value('update_thread_flag', {}) - elif content_type == 'comment': - self.set_mock_return_value('update_comment_flag', {}) + if content_type == "thread": + self.set_mock_return_value("update_thread_flag", {}) + elif content_type == "comment": + self.set_mock_return_value("update_comment_flag", {}) def register_read_response(self, user, content_type, content_id): - self.set_mock_return_value('mark_thread_as_read', {}) + self.set_mock_return_value("mark_thread_as_read", {}) def register_delete_thread_response(self, thread_id): - self.set_mock_return_value('delete_thread', {}) + self.set_mock_return_value("delete_thread", {}) def register_delete_comment_response(self, comment_id): - self.set_mock_return_value('delete_comment', {}) + self.set_mock_return_value("delete_comment", {}) def register_user_active_threads(self, user_id, response): - self.set_mock_return_value('get_user_active_threads', response) + self.set_mock_return_value("get_user_active_threads", response) def register_get_subscriptions(self, thread_id, response): - self.set_mock_return_value('get_thread_subscriptions', response) + self.set_mock_return_value("get_thread_subscriptions", response) def register_thread_flag_response(self, thread_id): """Register a mock response for PUT on the CS thread flag endpoints""" @@ -760,7 +806,7 @@ def request_patch(self, request_data): return self.client.patch( self.url, json.dumps(request_data), - content_type="application/merge-patch+json" + content_type="application/merge-patch+json", ) def expected_thread_data(self, overrides=None): @@ -818,6 +864,10 @@ def expected_thread_data(self, overrides=None): "close_reason": None, "close_reason_code": None, "learner_status": "new", + "is_deleted": None, + "deleted_at": None, + "deleted_by": None, + "deleted_by_label": None, } response_data.update(overrides or {}) return response_data @@ -890,7 +940,9 @@ def make_minimal_cs_comment(overrides=None): return ret -def make_paginated_api_response(results=None, count=0, num_pages=0, next_link=None, previous_link=None): +def make_paginated_api_response( + results=None, count=0, num_pages=0, next_link=None, previous_link=None +): """ Generates the response dictionary of paginated APIs with passed data """ @@ -901,7 +953,7 @@ def make_paginated_api_response(results=None, count=0, num_pages=0, next_link=No "count": count, "num_pages": num_pages, }, - "results": results or [] + "results": results or [], } @@ -919,7 +971,9 @@ def create_profile_image(self, user, storage): with make_image_file() as image_file: create_profile_images(image_file, get_profile_image_names(user.username)) self.check_images(user, storage) - set_has_profile_image(user.username, True, self.TEST_PROFILE_IMAGE_UPLOADED_AT) + set_has_profile_image( + user.username, True, self.TEST_PROFILE_IMAGE_UPLOADED_AT + ) def check_images(self, user, storage, exist=True): """ @@ -933,7 +987,7 @@ def check_images(self, user, storage, exist=True): assert storage.exists(name) with closing(Image.open(storage.path(name))) as img: assert img.size == (size, size) - assert img.format == 'JPEG' + assert img.format == "JPEG" else: assert not storage.exists(name) @@ -941,18 +995,18 @@ def get_expected_user_profile(self, username): """ Returns the expected user profile data for a given username """ - url = 'http://example-storage.com/profile-images/{filename}_{{size}}.jpg?v={timestamp}'.format( - filename=hashlib.md5(b'secret' + username.encode('utf-8')).hexdigest(), - timestamp=self.TEST_PROFILE_IMAGE_UPLOADED_AT.strftime("%s") + url = "http://example-storage.com/profile-images/{filename}_{{size}}.jpg?v={timestamp}".format( + filename=hashlib.md5(b"secret" + username.encode("utf-8")).hexdigest(), + timestamp=self.TEST_PROFILE_IMAGE_UPLOADED_AT.strftime("%s"), ) return { - 'profile': { - 'image': { - 'has_image': True, - 'image_url_full': url.format(size=500), - 'image_url_large': url.format(size=120), - 'image_url_medium': url.format(size=50), - 'image_url_small': url.format(size=30), + "profile": { + "image": { + "has_image": True, + "image_url_full": url.format(size=500), + "image_url_large": url.format(size=120), + "image_url_medium": url.format(size=50), + "image_url_small": url.format(size=30), } } } @@ -962,14 +1016,14 @@ def parsed_body(request): """Returns a parsed dictionary version of a request body""" # This could just be HTTPrettyRequest.parsed_body, but that method double-decodes '%2B' -> '+' -> ' '. # You can just remove this method when this issue is fixed: https://github.com/gabrielfalcao/HTTPretty/issues/240 - return parse_qs(request.body.decode('utf8')) + return parse_qs(request.body.decode("utf8")) def querystring(request): """Returns a parsed dictionary version of a query string""" # This could just be HTTPrettyRequest.querystring, but that method double-decodes '%2B' -> '+' -> ' '. # You can just remove this method when this issue is fixed: https://github.com/gabrielfalcao/HTTPretty/issues/240 - return parse_qs(request.path.split('?', 1)[-1]) + return parse_qs(request.path.split("?", 1)[-1]) class ThreadMock(object): @@ -977,7 +1031,9 @@ class ThreadMock(object): A mock thread object """ - def __init__(self, thread_id, creator, title, parent_id=None, body='', commentable_id=None): + def __init__( + self, thread_id, creator, title, parent_id=None, body="", commentable_id=None + ): self.id = thread_id self.user_id = str(creator.id) self.username = creator.username diff --git a/lms/djangoapps/discussion/rest_api/urls.py b/lms/djangoapps/discussion/rest_api/urls.py index f102dc41f249..9753774f075c 100644 --- a/lms/djangoapps/discussion/rest_api/urls.py +++ b/lms/djangoapps/discussion/rest_api/urls.py @@ -9,6 +9,7 @@ from lms.djangoapps.discussion.rest_api.views import ( BulkDeleteUserPosts, + BulkRestoreUserPosts, CommentViewSet, CourseActivityStatsView, CourseDiscussionRolesAPIView, @@ -18,8 +19,10 @@ CourseTopicsViewV3, CourseView, CourseViewV2, + DeletedContentView, LearnerThreadView, ReplaceUsernamesView, + RestoreContent, RetireUserView, ThreadViewSet, UploadFileView, @@ -31,26 +34,22 @@ urlpatterns = [ re_path( - r"^v1/courses/{}/settings$".format( - settings.COURSE_ID_PATTERN - ), + r"^v1/courses/{}/settings$".format(settings.COURSE_ID_PATTERN), CourseDiscussionSettingsAPIView.as_view(), name="discussion_course_settings", ), re_path( - r"^v1/courses/{}/learner/$".format( - settings.COURSE_ID_PATTERN - ), + r"^v1/courses/{}/learner/$".format(settings.COURSE_ID_PATTERN), LearnerThreadView.as_view(), name="discussion_learner_threads", ), re_path( - fr"^v1/courses/{settings.COURSE_KEY_PATTERN}/activity_stats", + rf"^v1/courses/{settings.COURSE_KEY_PATTERN}/activity_stats", CourseActivityStatsView.as_view(), name="discussion_course_activity_stats", ), re_path( - fr"^v1/courses/{settings.COURSE_ID_PATTERN}/upload$", + rf"^v1/courses/{settings.COURSE_ID_PATTERN}/upload$", UploadFileView.as_view(), name="upload_file", ), @@ -62,36 +61,55 @@ name="discussion_course_roles", ), re_path( - fr"^v1/courses/{settings.COURSE_ID_PATTERN}", + rf"^v1/courses/{settings.COURSE_ID_PATTERN}", CourseView.as_view(), - name="discussion_course" + name="discussion_course", ), re_path( - fr"^v2/courses/{settings.COURSE_ID_PATTERN}", + rf"^v2/courses/{settings.COURSE_ID_PATTERN}", CourseViewV2.as_view(), - name="discussion_course_v2" + name="discussion_course_v2", ), - re_path(r'^v1/accounts/retire_forum/?$', RetireUserView.as_view(), name="retire_discussion_user"), - path('v1/accounts/replace_username', ReplaceUsernamesView.as_view(), name="replace_discussion_username"), re_path( - fr"^v1/course_topics/{settings.COURSE_ID_PATTERN}", + r"^v1/accounts/retire_forum/?$", + RetireUserView.as_view(), + name="retire_discussion_user", + ), + path( + "v1/accounts/replace_username", + ReplaceUsernamesView.as_view(), + name="replace_discussion_username", + ), + re_path( + rf"^v1/course_topics/{settings.COURSE_ID_PATTERN}", CourseTopicsView.as_view(), - name="course_topics" + name="course_topics", ), re_path( - fr"^v2/course_topics/{settings.COURSE_ID_PATTERN}", + rf"^v2/course_topics/{settings.COURSE_ID_PATTERN}", CourseTopicsViewV2.as_view(), - name="course_topics_v2" + name="course_topics_v2", ), re_path( - fr"^v3/course_topics/{settings.COURSE_ID_PATTERN}", + rf"^v3/course_topics/{settings.COURSE_ID_PATTERN}", CourseTopicsViewV3.as_view(), - name="course_topics_v3" + name="course_topics_v3", ), re_path( - fr"^v1/bulk_delete_user_posts/{settings.COURSE_ID_PATTERN}", + rf"^v1/bulk_delete_user_posts/{settings.COURSE_ID_PATTERN}", BulkDeleteUserPosts.as_view(), - name="bulk_delete_user_posts" + name="bulk_delete_user_posts", + ), + re_path( + rf"^v1/bulk_restore_user_posts/{settings.COURSE_ID_PATTERN}", + BulkRestoreUserPosts.as_view(), + name="bulk_restore_user_posts", + ), + path("v1/restore_content", RestoreContent.as_view(), name="restore_content"), + re_path( + rf"^v1/deleted_content/{settings.COURSE_ID_PATTERN}", + DeletedContentView.as_view(), + name="deleted_content", ), - path('v1/', include(ROUTER.urls)), + path("v1/", include(ROUTER.urls)), ] diff --git a/lms/djangoapps/discussion/rest_api/views.py b/lms/djangoapps/discussion/rest_api/views.py index ba9818124e08..3556f78562fe 100644 --- a/lms/djangoapps/discussion/rest_api/views.py +++ b/lms/djangoapps/discussion/rest_api/views.py @@ -1,17 +1,19 @@ """ Discussion API views """ + import logging import uuid import edx_api_doc_tools as apidocs - from django.contrib.auth import get_user_model from django.core.exceptions import BadRequest, ValidationError from django.shortcuts import get_object_or_404 from drf_yasg import openapi from edx_rest_framework_extensions.auth.jwt.authentication import JwtAuthentication -from edx_rest_framework_extensions.auth.session.authentication import SessionAuthenticationAllowInactiveUser +from edx_rest_framework_extensions.auth.session.authentication import ( + SessionAuthenticationAllowInactiveUser, +) from opaque_keys.edx.keys import CourseKey from rest_framework import permissions, status from rest_framework.authentication import SessionAuthentication @@ -21,31 +23,49 @@ from rest_framework.views import APIView from rest_framework.viewsets import ViewSet -from xmodule.modulestore.django import modulestore - from common.djangoapps.student.models import CourseEnrollment from common.djangoapps.util.file import store_uploaded_file from lms.djangoapps.course_api.blocks.api import get_blocks from lms.djangoapps.course_goals.models import UserActivity +from lms.djangoapps.discussion.django_comment_client import settings as cc_settings +from lms.djangoapps.discussion.django_comment_client.utils import ( + get_group_id_for_comments_service, +) from lms.djangoapps.discussion.rate_limit import is_content_creation_rate_limited from lms.djangoapps.discussion.rest_api.permissions import IsAllowedToBulkDelete -from lms.djangoapps.discussion.rest_api.tasks import delete_course_post_for_user +from lms.djangoapps.discussion.rest_api.tasks import ( + delete_course_post_for_user, + restore_course_post_for_user, +) from lms.djangoapps.discussion.toggles import ONLY_VERIFIED_USERS_CAN_POST -from lms.djangoapps.discussion.django_comment_client import settings as cc_settings -from lms.djangoapps.discussion.django_comment_client.utils import get_group_id_for_comments_service from lms.djangoapps.instructor.access import update_forum_role -from openedx.core.djangoapps.discussions.config.waffle import ENABLE_NEW_STRUCTURE_DISCUSSIONS -from openedx.core.djangoapps.discussions.models import DiscussionsConfiguration, Provider +from openedx.core.djangoapps.discussions.config.waffle import ( + ENABLE_NEW_STRUCTURE_DISCUSSIONS, +) +from openedx.core.djangoapps.discussions.models import ( + DiscussionsConfiguration, + Provider, +) from openedx.core.djangoapps.discussions.serializers import DiscussionSettingsSerializer from openedx.core.djangoapps.django_comment_common import comment_client -from openedx.core.djangoapps.django_comment_common.models import CourseDiscussionSettings, Role from openedx.core.djangoapps.django_comment_common.comment_client.comment import Comment from openedx.core.djangoapps.django_comment_common.comment_client.thread import Thread -from openedx.core.djangoapps.user_api.accounts.permissions import CanReplaceUsername, CanRetireUser +from openedx.core.djangoapps.django_comment_common.models import ( + CourseDiscussionSettings, + Role, +) +from openedx.core.djangoapps.user_api.accounts.permissions import ( + CanReplaceUsername, + CanRetireUser, +) from openedx.core.djangoapps.user_api.models import UserRetirementStatus -from openedx.core.lib.api.authentication import BearerAuthentication, BearerAuthenticationAllowInactiveUser +from openedx.core.lib.api.authentication import ( + BearerAuthentication, + BearerAuthenticationAllowInactiveUser, +) from openedx.core.lib.api.parsers import MergePatchParser from openedx.core.lib.api.view_utils import DeveloperErrorViewMixin, view_auth_classes +from xmodule.modulestore.django import modulestore from ..rest_api.api import ( create_comment, @@ -57,10 +77,10 @@ get_course_discussion_user_stats, get_course_topics, get_course_topics_v2, + get_learner_active_thread_list, get_response_comments, get_thread, get_thread_list, - get_learner_active_thread_list, get_user_comments, get_v2_course_topics_as_v1, update_comment, @@ -88,10 +108,10 @@ from .utils import ( create_blocks_params, create_topics_v3_structure, - is_captcha_enabled, - verify_recaptcha_token, get_course_id_from_thread_id, + is_captcha_enabled, is_only_student, + verify_recaptcha_token, ) log = logging.getLogger(__name__) @@ -107,14 +127,16 @@ class CourseView(DeveloperErrorViewMixin, APIView): @apidocs.schema( parameters=[ - apidocs.string_parameter("course_id", apidocs.ParameterLocation.PATH, description="Course ID") + apidocs.string_parameter( + "course_id", apidocs.ParameterLocation.PATH, description="Course ID" + ) ], responses={ 200: CourseMetadataSerailizer(read_only=True, required=False), 401: "The requester is not authenticated.", 403: "The requester cannot access the specified course.", 404: "The requested course does not exist.", - } + }, ) def get(self, request, course_id): """ @@ -126,7 +148,9 @@ def get(self, request, course_id): """ course_key = CourseKey.from_string(course_id) # TODO: which class is right? # Record user activity for tracking progress towards a user's course goals (for mobile app) - UserActivity.record_user_activity(request.user, course_key, request=request, only_if_mobile_app=True) + UserActivity.record_user_activity( + request.user, course_key, request=request, only_if_mobile_app=True + ) return Response(get_course(request, course_key)) @@ -138,14 +162,16 @@ class CourseViewV2(DeveloperErrorViewMixin, APIView): @apidocs.schema( parameters=[ - apidocs.string_parameter("course_id", apidocs.ParameterLocation.PATH, description="Course ID") + apidocs.string_parameter( + "course_id", apidocs.ParameterLocation.PATH, description="Course ID" + ) ], responses={ 200: CourseMetadataSerailizer(read_only=True, required=False), 401: "The requester is not authenticated.", 403: "The requester cannot access the specified course.", 404: "The requested course does not exist.", - } + }, ) def get(self, request, course_id): """ @@ -156,7 +182,9 @@ def get(self, request, course_id): """ course_key = CourseKey.from_string(course_id) # Record user activity for tracking progress towards a user's course goals (for mobile app) - UserActivity.record_user_activity(request.user, course_key, request=request, only_if_mobile_app=True) + UserActivity.record_user_activity( + request.user, course_key, request=request, only_if_mobile_app=True + ) return Response(get_course(request, course_key, False)) @@ -221,14 +249,14 @@ def get(self, request, course_key_string): form_query_string = CourseActivityStatsForm(request.query_params) if not form_query_string.is_valid(): raise ValidationError(form_query_string.errors) - order_by = form_query_string.cleaned_data.get('order_by', None) + order_by = form_query_string.cleaned_data.get("order_by", None) order_by = UserOrdering(order_by) if order_by else None - username_search_string = form_query_string.cleaned_data.get('username', None) + username_search_string = form_query_string.cleaned_data.get("username", None) data = get_course_discussion_user_stats( request, course_key_string, - form_query_string.cleaned_data['page'], - form_query_string.cleaned_data['page_size'], + form_query_string.cleaned_data["page"], + form_query_string.cleaned_data["page_size"], order_by, username_search_string, ) @@ -268,19 +296,17 @@ def get(self, request, course_id): Implements the GET method as described in the class docstring. """ course_key = CourseKey.from_string(course_id) - topic_ids = self.request.GET.get('topic_id') - topic_ids = set(topic_ids.strip(',').split(',')) if topic_ids else None + topic_ids = self.request.GET.get("topic_id") + topic_ids = set(topic_ids.strip(",").split(",")) if topic_ids else None with modulestore().bulk_operations(course_key): configuration = DiscussionsConfiguration.get(context_key=course_key) provider = configuration.provider_type # This will be removed when mobile app will support new topic structure - new_structure_enabled = ENABLE_NEW_STRUCTURE_DISCUSSIONS.is_enabled(course_key) + new_structure_enabled = ENABLE_NEW_STRUCTURE_DISCUSSIONS.is_enabled( + course_key + ) if provider == Provider.OPEN_EDX and new_structure_enabled: - response = get_v2_course_topics_as_v1( - request, - course_key, - topic_ids - ) + response = get_v2_course_topics_as_v1(request, course_key, topic_ids) else: response = get_course_topics( request, @@ -288,7 +314,9 @@ def get(self, request, course_id): topic_ids, ) # Record user activity for tracking progress towards a user's course goals (for mobile app) - UserActivity.record_user_activity(request.user, course_key, request=request, only_if_mobile_app=True) + UserActivity.record_user_activity( + request.user, course_key, request=request, only_if_mobile_app=True + ) return Response(response) @@ -304,17 +332,17 @@ class CourseTopicsViewV2(DeveloperErrorViewMixin, APIView): @apidocs.schema( parameters=[ apidocs.string_parameter( - 'course_id', + "course_id", apidocs.ParameterLocation.PATH, description="Course ID", ), apidocs.string_parameter( - 'topic_id', + "topic_id", apidocs.ParameterLocation.QUERY, description="Comma-separated list of topic ids to filter", ), openapi.Parameter( - 'order_by', + "order_by", apidocs.ParameterLocation.QUERY, required=False, type=openapi.TYPE_STRING, @@ -327,7 +355,7 @@ class CourseTopicsViewV2(DeveloperErrorViewMixin, APIView): 401: "The requester is not authenticated.", 403: "The requester cannot access the specified course.", 404: "The requested course does not exist.", - } + }, ) def get(self, request, course_id): """ @@ -348,7 +376,7 @@ def get(self, request, course_id): course_key, request.user, form_query_params.cleaned_data["topic_id"], - form_query_params.cleaned_data["order_by"] + form_query_params.cleaned_data["order_by"], ) return Response(response) @@ -416,17 +444,17 @@ def get(self, request, course_id): blocks_params = create_blocks_params(course_usage_key, request.user) blocks = get_blocks( request, - blocks_params['usage_key'], - blocks_params['user'], - blocks_params['depth'], - blocks_params['nav_depth'], - blocks_params['requested_fields'], - blocks_params['block_counts'], - blocks_params['student_view_data'], - blocks_params['return_type'], - blocks_params['block_types_filter'], + blocks_params["usage_key"], + blocks_params["user"], + blocks_params["depth"], + blocks_params["nav_depth"], + blocks_params["requested_fields"], + blocks_params["block_counts"], + blocks_params["student_view_data"], + blocks_params["return_type"], + blocks_params["block_types_filter"], hide_access_denials=False, - )['blocks'] + )["blocks"] topics = create_topics_v3_structure(blocks, topics) return Response(topics) @@ -627,8 +655,12 @@ class ThreadViewSet(DeveloperErrorViewMixin, ViewSet): No content is returned for a DELETE request """ + lookup_field = "thread_id" - parser_classes = (JSONParser, MergePatchParser,) + parser_classes = ( + JSONParser, + MergePatchParser, + ) def list(self, request): """ @@ -641,7 +673,10 @@ class docstring. # Record user activity for tracking progress towards a user's course goals (for mobile app) UserActivity.record_user_activity( - request.user, form.cleaned_data["course_id"], request=request, only_if_mobile_app=True + request.user, + form.cleaned_data["course_id"], + request=request, + only_if_mobile_app=True, ) return get_thread_list( @@ -660,14 +695,15 @@ class docstring. form.cleaned_data["order_direction"], form.cleaned_data["requested_fields"], form.cleaned_data["count_flagged"], + form.cleaned_data["show_deleted"], ) def retrieve(self, request, thread_id=None): """ Implements the GET method for thread ID """ - requested_fields = request.GET.get('requested_fields') - course_id = request.GET.get('course_id') + requested_fields = request.GET.get("requested_fields") + course_id = request.GET.get("course_id") return Response(get_thread(request, thread_id, requested_fields, course_id)) def create(self, request): @@ -681,21 +717,28 @@ class docstring. course_key = CourseKey.from_string(course_key_str) if is_content_creation_rate_limited(request, course_key=course_key): - return Response("Too many requests", status=status.HTTP_429_TOO_MANY_REQUESTS) + return Response( + "Too many requests", status=status.HTTP_429_TOO_MANY_REQUESTS + ) if is_captcha_enabled(course_key) and is_only_student(course_key, request.user): - captcha_token = request.data.get('captcha_token') + captcha_token = request.data.get("captcha_token") if not captcha_token: - raise ValidationError({'captcha_token': 'This field is required.'}) + raise ValidationError({"captcha_token": "This field is required."}) if not verify_recaptcha_token(captcha_token): - return Response({'error': 'CAPTCHA verification failed.'}, status=400) - - if ONLY_VERIFIED_USERS_CAN_POST.is_enabled(course_key) and not request.user.is_active: - raise ValidationError({"detail": "Only verified users can post in discussions."}) + return Response({"error": "CAPTCHA verification failed."}, status=400) + + if ( + ONLY_VERIFIED_USERS_CAN_POST.is_enabled(course_key) + and not request.user.is_active + ): + raise ValidationError( + {"detail": "Only verified users can post in discussions."} + ) data = request.data.copy() - data.pop('captcha_token', None) + data.pop("captcha_token", None) return Response(create_thread(request, data)) def partial_update(self, request, thread_id): @@ -762,24 +805,27 @@ def get(self, request, course_id=None): Implements the GET method as described in the class docstring. """ course_key = CourseKey.from_string(course_id) - page_num = request.GET.get('page', 1) - threads_per_page = request.GET.get('page_size', 10) - count_flagged = request.GET.get('count_flagged', False) - thread_type = request.GET.get('thread_type') - order_by = request.GET.get('order_by') + page_num = request.GET.get("page", 1) + threads_per_page = request.GET.get("page_size", 10) + count_flagged = request.GET.get("count_flagged", False) + thread_type = request.GET.get("thread_type") + order_by = request.GET.get("order_by") order_by_mapping = { "last_activity_at": "activity", "comment_count": "comments", - "vote_count": "votes" + "vote_count": "votes", } - order_by = order_by_mapping.get(order_by, 'activity') - post_status = request.GET.get('status', None) + order_by = order_by_mapping.get(order_by, "activity") + post_status = request.GET.get("status", None) + show_deleted = request.GET.get("show_deleted", "false").lower() == "true" discussion_id = None - username = request.GET.get('username', None) + username = request.GET.get("username", None) user = get_object_or_404(User, username=username) group_id = None try: - group_id = get_group_id_for_comments_service(request, course_key, discussion_id) + group_id = get_group_id_for_comments_service( + request, course_key, discussion_id + ) except ValueError: pass @@ -792,14 +838,17 @@ def get(self, request, course_id=None): "count_flagged": count_flagged, "thread_type": thread_type, "sort_key": order_by, + "show_deleted": show_deleted, } if post_status: - if post_status not in ['flagged', 'unanswered', 'unread', 'unresponded']: - raise ValidationError({ - "status": [ - f"Invalid value. '{post_status}' must be 'flagged', 'unanswered', 'unread' or 'unresponded" - ] - }) + if post_status not in ["flagged", "unanswered", "unread", "unresponded"]: + raise ValidationError( + { + "status": [ + f"Invalid value. '{post_status}' must be 'flagged', 'unanswered', 'unread' or 'unresponded" + ] + } + ) query_params[post_status] = True return get_learner_active_thread_list(request, course_key, query_params) @@ -968,8 +1017,12 @@ class CommentViewSet(DeveloperErrorViewMixin, ViewSet): No content is returned for a DELETE request """ + lookup_field = "comment_id" - parser_classes = (JSONParser, MergePatchParser,) + parser_classes = ( + JSONParser, + MergePatchParser, + ) def list(self, request): """ @@ -1010,7 +1063,8 @@ def list_by_thread(self, request): form.cleaned_data["page_size"], form.cleaned_data["flagged"], form.cleaned_data["requested_fields"], - form.cleaned_data["merge_question_type_responses"] + form.cleaned_data["merge_question_type_responses"], + form.cleaned_data["show_deleted"], ) def list_by_user(self, request): @@ -1057,21 +1111,28 @@ class docstring. course_key = CourseKey.from_string(course_key_str) if is_content_creation_rate_limited(request, course_key=course_key): - return Response("Too many requests", status=status.HTTP_429_TOO_MANY_REQUESTS) + return Response( + "Too many requests", status=status.HTTP_429_TOO_MANY_REQUESTS + ) if is_captcha_enabled(course_key) and is_only_student(course_key, request.user): - captcha_token = request.data.get('captcha_token') + captcha_token = request.data.get("captcha_token") if not captcha_token: - raise ValidationError({'captcha_token': 'This field is required.'}) + raise ValidationError({"captcha_token": "This field is required."}) if not verify_recaptcha_token(captcha_token): - return Response({'error': 'CAPTCHA verification failed.'}, status=400) - - if ONLY_VERIFIED_USERS_CAN_POST.is_enabled(course_key) and not request.user.is_active: - raise ValidationError({"detail": "Only verified users can post in discussions."}) + return Response({"error": "CAPTCHA verification failed."}, status=400) + + if ( + ONLY_VERIFIED_USERS_CAN_POST.is_enabled(course_key) + and not request.user.is_active + ): + raise ValidationError( + {"detail": "Only verified users can post in discussions."} + ) data = request.data.copy() - data.pop('captcha_token', None) + data.pop("captcha_token", None) return Response(create_comment(request, data)) def destroy(self, request, comment_id): @@ -1147,8 +1208,11 @@ def post(self, request, course_id): unique_file_name = f"{course_id}/{thread_key}/{uuid.uuid4()}" try: file_storage, stored_file_name = store_uploaded_file( - request, "uploaded_file", cc_settings.ALLOWED_UPLOAD_FILE_TYPES, - unique_file_name, max_file_size=cc_settings.MAX_UPLOAD_FILE_SIZE, + request, + "uploaded_file", + cc_settings.ALLOWED_UPLOAD_FILE_TYPES, + unique_file_name, + max_file_size=cc_settings.MAX_UPLOAD_FILE_SIZE, ) except ValueError as err: raise BadRequest("no `uploaded_file` was provided") from err @@ -1189,10 +1253,12 @@ def post(self, request): """ Implements the retirement endpoint. """ - username = request.data['username'] + username = request.data["username"] try: - retirement = UserRetirementStatus.get_retirement_for_retirement_action(username) + retirement = UserRetirementStatus.get_retirement_for_retirement_action( + username + ) cc_user = comment_client.User.from_django_user(retirement.user) # Send the retired username to the forums service, as the service cannot generate @@ -1247,7 +1313,9 @@ def post(self, request): for username_pair in username_mappings: current_username = list(username_pair.keys())[0] new_username = list(username_pair.values())[0] - successfully_replaced = self._replace_username(current_username, new_username) + successfully_replaced = self._replace_username( + current_username, new_username + ) if successfully_replaced: successful_replacements.append({current_username: new_username}) else: @@ -1257,8 +1325,8 @@ def post(self, request): status=status.HTTP_200_OK, data={ "successful_replacements": successful_replacements, - "failed_replacements": failed_replacements - } + "failed_replacements": failed_replacements, + }, ) def _replace_username(self, current_username, new_username): @@ -1304,7 +1372,7 @@ def _replace_username(self, current_username, new_username): return True def _has_valid_schema(self, post_data): - """ Verifies the data is a list of objects with a single key:value pair """ + """Verifies the data is a list of objects with a single key:value pair""" if not isinstance(post_data, list): return False for obj in post_data: @@ -1364,12 +1432,16 @@ class CourseDiscussionSettingsAPIView(DeveloperErrorViewMixin, APIView): * available_division_schemes: A list of available division schemes for the course. """ + authentication_classes = ( JwtAuthentication, BearerAuthenticationAllowInactiveUser, SessionAuthenticationAllowInactiveUser, ) - parser_classes = (JSONParser, MergePatchParser,) + parser_classes = ( + JSONParser, + MergePatchParser, + ) permission_classes = (permissions.IsAuthenticated, IsStaffOrAdmin) def _get_request_kwargs(self, course_id): @@ -1385,14 +1457,14 @@ def get(self, request, course_id): if not form.is_valid(): raise ValidationError(form.errors) - course_key = form.cleaned_data['course_key'] - course = form.cleaned_data['course'] + course_key = form.cleaned_data["course_key"] + course = form.cleaned_data["course"] discussion_settings = CourseDiscussionSettings.get(course_key) serializer = DiscussionSettingsSerializer( discussion_settings, context={ - 'course': course, - 'settings': discussion_settings, + "course": course, + "settings": discussion_settings, }, partial=True, ) @@ -1411,15 +1483,15 @@ def patch(self, request, course_id): if not form.is_valid(): raise ValidationError(form.errors) - course = form.cleaned_data['course'] - course_key = form.cleaned_data['course_key'] + course = form.cleaned_data["course"] + course_key = form.cleaned_data["course_key"] discussion_settings = CourseDiscussionSettings.get(course_key) serializer = DiscussionSettingsSerializer( discussion_settings, context={ - 'course': course, - 'settings': discussion_settings, + "course": course, + "settings": discussion_settings, }, data=request.data, partial=True, @@ -1488,6 +1560,7 @@ class CourseDiscussionRolesAPIView(DeveloperErrorViewMixin, APIView): * division_scheme: The division scheme used by the course. """ + authentication_classes = ( JwtAuthentication, BearerAuthenticationAllowInactiveUser, @@ -1508,11 +1581,13 @@ def get(self, request, course_id, rolename): if not form.is_valid(): raise ValidationError(form.errors) - course_id = form.cleaned_data['course_key'] - role = form.cleaned_data['role'] + course_id = form.cleaned_data["course_key"] + role = form.cleaned_data["role"] - data = {'course_id': course_id, 'users': role.users.all()} - context = {'course_discussion_settings': CourseDiscussionSettings.get(course_id)} + data = {"course_id": course_id, "users": role.users.all()} + context = { + "course_discussion_settings": CourseDiscussionSettings.get(course_id) + } serializer = DiscussionRolesListSerializer(data, context=context) return Response(serializer.data) @@ -1526,23 +1601,25 @@ def post(self, request, course_id, rolename): if not form.is_valid(): raise ValidationError(form.errors) - course_id = form.cleaned_data['course_key'] - rolename = form.cleaned_data['rolename'] + course_id = form.cleaned_data["course_key"] + rolename = form.cleaned_data["rolename"] serializer = DiscussionRolesSerializer(data=request.data) if not serializer.is_valid(): raise ValidationError(serializer.errors) - action = serializer.validated_data['action'] - user = serializer.validated_data['user'] + action = serializer.validated_data["action"] + user = serializer.validated_data["user"] try: update_forum_role(course_id, user, rolename, action) except Role.DoesNotExist as err: raise ValidationError(f"Role '{rolename}' does not exist") from err - role = form.cleaned_data['role'] - data = {'course_id': course_id, 'users': role.users.all()} - context = {'course_discussion_settings': CourseDiscussionSettings.get(course_id)} + role = form.cleaned_data["role"] + data = {"course_id": course_id, "users": role.users.all()} + context = { + "course_discussion_settings": CourseDiscussionSettings.get(course_id) + } serializer = DiscussionRolesListSerializer(data, context=context) return Response(serializer.data) @@ -1566,7 +1643,9 @@ class BulkDeleteUserPosts(DeveloperErrorViewMixin, APIView): """ authentication_classes = ( - JwtAuthentication, BearerAuthentication, SessionAuthentication, + JwtAuthentication, + BearerAuthentication, + SessionAuthentication, ) permission_classes = (permissions.IsAuthenticated, IsAllowedToBulkDelete) @@ -1587,23 +1666,26 @@ def post(self, request, course_id): course_ids = [course_id] if course_or_org == "org": org_id = CourseKey.from_string(course_id).org - enrollments = CourseEnrollment.objects.filter(user=request.user).values_list('course_id', flat=True) - course_ids.extend([ - str(c_id) - for c_id in enrollments - if c_id.org == org_id - ]) + enrollments = CourseEnrollment.objects.filter( + user=request.user + ).values_list("course_id", flat=True) + course_ids.extend([str(c_id) for c_id in enrollments if c_id.org == org_id]) course_ids = list(set(course_ids)) log.info(f"<> {username} enrolled in {enrollments}") - log.info(f"<> Posts for {username} in {course_ids} - for {course_or_org} {course_id}") + log.info( + f"<> Posts for {username} in {course_ids} - for {course_or_org} {course_id}" + ) comment_count = Comment.get_user_comment_count(user.id, course_ids) thread_count = Thread.get_user_threads_count(user.id, course_ids) - log.info(f"<> {username} in {course_ids} - Count thread {thread_count}, comment {comment_count}") + log.info( + f"<> {username} in {course_ids} - Count thread {thread_count}, comment {comment_count}" + ) if execute_task: event_data = { "triggered_by": request.user.username, + "triggered_by_user_id": str(request.user.id), "username": username, "course_or_org": course_or_org, "course_key": course_id, @@ -1613,5 +1695,256 @@ def post(self, request, course_id): ) return Response( {"comment_count": comment_count, "thread_count": thread_count}, - status=status.HTTP_202_ACCEPTED + status=status.HTTP_202_ACCEPTED, + ) + + +class RestoreContent(DeveloperErrorViewMixin, APIView): + """ + **Use Cases** + A privileged user that can restore individual soft-deleted threads, comments, or responses. + + **Example Requests**: + POST /api/discussion/v1/restore_content + Request Body: + { + "content_type": "thread", // "thread", "comment", or "response" + "content_id": "thread_id_or_comment_id", + "course_id": "course-v1:edX+DemoX+Demo_Course" + } + + **Example Response**: + {"success": true, "message": "Content restored successfully"} + """ + + authentication_classes = ( + JwtAuthentication, + BearerAuthentication, + SessionAuthentication, + ) + permission_classes = (permissions.IsAuthenticated, IsAllowedToBulkDelete) + + def post(self, request): + """ + Implements the restore individual content endpoint. + """ + content_type = request.data.get("content_type") + content_id = request.data.get("content_id") + course_id = request.data.get("course_id") + + if not all([content_type, content_id, course_id]): + raise BadRequest("content_type, content_id, and course_id are required.") + + if content_type not in ["thread", "comment", "response"]: + raise BadRequest("content_type must be 'thread', 'comment', or 'response'.") + + restored_by_user_id = str(request.user.id) + + try: + if content_type == "thread": + success = Thread.restore_thread( + content_id, course_id=course_id, restored_by=restored_by_user_id + ) + else: # comment or response (both are comments in the backend) + success = Comment.restore_comment( + content_id, course_id=course_id, restored_by=restored_by_user_id + ) + + if success: + return Response( + { + "success": True, + "message": f"{content_type.capitalize()} restored successfully", + }, + status=status.HTTP_200_OK, + ) + else: + return Response( + { + "success": False, + "message": f"{content_type.capitalize()} not found or already restored", + }, + status=status.HTTP_404_NOT_FOUND, + ) + except Exception as e: # pylint: disable=broad-exception-caught + log.error("Error restoring %s %s: %s", content_type, content_id, str(e)) + return Response( + { + "success": False, + "message": f"Error restoring {content_type}: {str(e)}", + }, + status=status.HTTP_500_INTERNAL_SERVER_ERROR, + ) + + +class BulkRestoreUserPosts(DeveloperErrorViewMixin, APIView): + """ + **Use Cases** + A privileged user that can restore all soft-deleted posts and comments made by a user. + It returns expected number of comments and threads that will be restored + + **Example Requests**: + POST /api/discussion/v1/bulk_restore_user_posts/{course_id} + Query Parameters: + username: The username of the user whose posts are to be restored + course_id: Course id for which posts are to be restored + execute: If True, runs restoration task + course_or_org: If 'course', restores posts in the course, if 'org', restores posts in all courses of the org + + **Example Response**: + {"comment_count": 5, "thread_count": 3} + """ + + authentication_classes = ( + JwtAuthentication, + BearerAuthentication, + SessionAuthentication, + ) + permission_classes = (permissions.IsAuthenticated, IsAllowedToBulkDelete) + + def post(self, request, course_id): + """ + Implements the restore user posts endpoint. + """ + username = request.GET.get("username", None) + execute_task = request.GET.get("execute", "false").lower() == "true" + if (not username) or (not course_id): + raise BadRequest("username and course_id are required.") + course_or_org = request.GET.get("course_or_org", "course") + if course_or_org not in ["course", "org"]: + raise BadRequest("course_or_org must be either 'course' or 'org'.") + + user = get_object_or_404(User, username=username) + course_ids = [course_id] + if course_or_org == "org": + org_id = CourseKey.from_string(course_id).org + enrollments = CourseEnrollment.objects.filter( + user=request.user + ).values_list("course_id", flat=True) + course_ids.extend([str(c_id) for c_id in enrollments if c_id.org == org_id]) + course_ids = list(set(course_ids)) + log.info("<> %s enrolled in %s", username, enrollments) + log.info( + "<> Posts for %s in %s - for %s %s", + username, + course_ids, + course_or_org, + course_id, + ) + + comment_count = Comment.get_user_deleted_comment_count(user.id, course_ids) + thread_count = Thread.get_user_deleted_threads_count(user.id, course_ids) + log.info( + "<> %s in %s - Count thread %s, comment %s", + username, + course_ids, + thread_count, + comment_count, + ) + + if execute_task: + event_data = { + "triggered_by": request.user.username, + "triggered_by_user_id": str(request.user.id), + "username": username, + "course_or_org": course_or_org, + "course_key": course_id, + } + restore_course_post_for_user.apply_async( + args=(user.id, username, course_ids, event_data), + ) + return Response( + {"comment_count": comment_count, "thread_count": thread_count}, + status=status.HTTP_202_ACCEPTED, ) + + +class DeletedContentView(DeveloperErrorViewMixin, APIView): + """ + **Use Cases** + Retrieve all deleted content (threads, comments, responses) for a course. + This endpoint allows privileged users to fetch deleted discussion content. + + **Example Requests**: + GET /api/discussion/v1/deleted_content/course-v1:edX+DemoX+Demo_Course + GET /api/discussion/v1/deleted_content/course-v1:edX+DemoX+Demo_Course?content_type=thread + GET /api/discussion/v1/deleted_content/course-v1:edX+DemoX+Demo_Course?page=1&per_page=20 + + **Example Response**: + { + "results": [ + { + "id": "thread_id", + "type": "thread", + "title": "Deleted Thread Title", + "body": "Thread content...", + "course_id": "course-v1:edX+DemoX+Demo_Course", + "author_id": "user_123", + "deleted_at": "2023-11-19T10:30:00Z", + "deleted_by": "moderator_456" + } + ], + "pagination": { + "page": 1, + "per_page": 20, + "total_count": 50, + "num_pages": 3 + } + } + """ + + authentication_classes = ( + JwtAuthentication, + BearerAuthentication, + SessionAuthentication, + ) + permission_classes = (permissions.IsAuthenticated, IsAllowedToBulkDelete) + + def get(self, request, course_id): + """ + Retrieve all deleted content for a course. + """ + try: + course_key = CourseKey.from_string(course_id) + except Exception as e: + raise BadRequest("Invalid course_id") from e + + # Get query parameters + content_type = request.GET.get( + "content_type", None + ) # 'thread', 'comment', or None for all + page = int(request.GET.get("page", 1)) + per_page = int(request.GET.get("per_page", 20)) + author_id = request.GET.get("author_id", None) + + # Validate parameters + if content_type and content_type not in ["thread", "comment"]: + raise BadRequest("content_type must be 'thread' or 'comment'") + + per_page = min(per_page, 100) # Limit to prevent excessive load + + try: + # Import here to avoid circular imports + from lms.djangoapps.discussion.rest_api.api import ( + get_deleted_content_for_course, + ) + + results = get_deleted_content_for_course( + request=request, + course_id=str(course_key), + content_type=content_type, + page=page, + per_page=per_page, + author_id=author_id, + ) + + return Response(results, status=status.HTTP_200_OK) + + except Exception as e: # pylint: disable=broad-exception-caught + logging.exception( + "Error retrieving deleted content for course %s: %s", course_id, e + ) + return Response( + {"error": "Failed to retrieve deleted content"}, + status=status.HTTP_500_INTERNAL_SERVER_ERROR, + ) diff --git a/openedx/core/djangoapps/django_comment_common/comment_client/comment.py b/openedx/core/djangoapps/django_comment_common/comment_client/comment.py index 8905679a45db..dae2088594ae 100644 --- a/openedx/core/djangoapps/django_comment_common/comment_client/comment.py +++ b/openedx/core/djangoapps/django_comment_common/comment_client/comment.py @@ -4,13 +4,17 @@ from bs4 import BeautifulSoup -from openedx.core.djangoapps.django_comment_common.comment_client import models, settings +from forum import api as forum_api +from forum.backends.mongodb.comments import ( + Comment as ForumComment, +) # pylint: disable=import-error +from openedx.core.djangoapps.django_comment_common.comment_client import ( + models, + settings, +) from .thread import Thread from .utils import CommentClientRequestError, get_course_key -from forum import api as forum_api -from forum.backends.mongodb.comments import Comment as ForumComment - log = logging.getLogger(__name__) @@ -18,26 +22,56 @@ class Comment(models.Model): accessible_fields = [ - 'id', 'body', 'anonymous', 'anonymous_to_peers', 'course_id', - 'endorsed', 'parent_id', 'thread_id', 'username', 'votes', 'user_id', - 'closed', 'created_at', 'updated_at', 'depth', 'at_position_list', - 'type', 'commentable_id', 'abuse_flaggers', 'endorsement', - 'child_count', 'edit_history', - 'is_spam', 'ai_moderation_reason', 'abuse_flagged', + "id", + "body", + "anonymous", + "anonymous_to_peers", + "course_id", + "endorsed", + "parent_id", + "thread_id", + "username", + "votes", + "user_id", + "closed", + "created_at", + "updated_at", + "depth", + "at_position_list", + "type", + "commentable_id", + "abuse_flaggers", + "endorsement", + "child_count", + "edit_history", + "is_spam", + "ai_moderation_reason", + "abuse_flagged", + "is_deleted", + "deleted_at", + "deleted_by", ] updatable_fields = [ - 'body', 'anonymous', 'anonymous_to_peers', 'course_id', 'closed', - 'user_id', 'endorsed', 'endorsement_user_id', 'edit_reason_code', - 'closing_user_id', 'editing_user_id', + "body", + "anonymous", + "anonymous_to_peers", + "course_id", + "closed", + "user_id", + "endorsed", + "endorsement_user_id", + "edit_reason_code", + "closing_user_id", + "editing_user_id", ] initializable_fields = updatable_fields - metrics_tag_fields = ['course_id', 'endorsed', 'closed'] + metrics_tag_fields = ["course_id", "endorsed", "closed"] base_url = f"{settings.PREFIX}/comments" - type = 'comment' + type = "comment" def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) @@ -46,7 +80,7 @@ def __init__(self, *args, **kwargs): @property def thread(self): if not self._cached_thread: - self._cached_thread = Thread(id=self.thread_id, type='thread') + self._cached_thread = Thread(id=self.thread_id, type="thread") return self._cached_thread @property @@ -56,22 +90,22 @@ def context(self): @classmethod def url_for_comments(cls, params=None): - if params and params.get('parent_id'): - return _url_for_comment(params['parent_id']) + if params and params.get("parent_id"): + return _url_for_comment(params["parent_id"]) else: - return _url_for_thread_comments(params['thread_id']) + return _url_for_thread_comments(params["thread_id"]) @classmethod def url(cls, action, params=None): if params is None: params = {} - if action in ['post']: + if action in ["post"]: return cls.url_for_comments(params) else: return super().url(action, params) def flagAbuse(self, user, voteable, course_id=None): - if voteable.type != 'comment': + if voteable.type != "comment": raise CommentClientRequestError("Can only flag comments") course_key = get_course_key(self.attributes.get("course_id") or course_id) @@ -84,7 +118,7 @@ def flagAbuse(self, user, voteable, course_id=None): voteable._update_from_response(response) def unFlagAbuse(self, user, voteable, removeAll, course_id=None): - if voteable.type != 'comment': + if voteable.type != "comment": raise CommentClientRequestError("Can only unflag comments") course_key = get_course_key(self.attributes.get("course_id") or course_id) @@ -102,7 +136,7 @@ def body_text(self): """ Return the text content of the comment html body. """ - soup = BeautifulSoup(self.body, 'html.parser') + soup = BeautifulSoup(self.body, "html.parser") return soup.get_text() @classmethod @@ -114,12 +148,15 @@ def get_user_comment_count(cls, user_id, course_ids): query_params = { "course_id": {"$in": course_ids}, "author_id": str(user_id), - "_type": "Comment" + "is_deleted": {"$ne": True}, + "_type": "Comment", } - return ForumComment()._collection.count_documents(query_params) # pylint: disable=protected-access + return ForumComment()._collection.count_documents( + query_params + ) # pylint: disable=protected-access @classmethod - def delete_user_comments(cls, user_id, course_ids): + def delete_user_comments(cls, user_id, course_ids, deleted_by=None): """ Deletes comments and responses of user in the given course_ids. TODO: Add support for MySQL backend as well @@ -128,21 +165,66 @@ def delete_user_comments(cls, user_id, course_ids): query_params = { "course_id": {"$in": course_ids}, "author_id": str(user_id), + "is_deleted": {"$ne": True}, } comments_deleted = 0 comments = ForumComment().get_list(**query_params) - log.info(f"<> Fetched comments for user {user_id} in {time.time() - start_time} seconds") + log.info( + f"<> Fetched comments for user {user_id} in {time.time() - start_time} seconds" + ) for comment in comments: start_time = time.time() comment_id = comment.get("_id") course_id = comment.get("course_id") if comment_id: - forum_api.delete_comment(comment_id, course_id=course_id) + # Use forum_api.delete_comment which supports deleted_by parameter + forum_api.delete_comment( # pylint: disable=unexpected-keyword-arg + comment_id, course_id=course_id, deleted_by=deleted_by + ) comments_deleted += 1 - log.info(f"<> Deleted comment {comment_id} in {time.time() - start_time} seconds." - f" Comment Found: {comment_id is not None}") + log.info( + f"<> Deleted comment {comment_id} in {time.time() - start_time} seconds." + f" Comment Found: {comment_id is not None}" + ) return comments_deleted + @classmethod + def get_user_deleted_comment_count(cls, user_id, course_ids): + """ + Returns count of deleted comments for user in the given course_ids. + """ + query_params = { + "course_id": {"$in": course_ids}, + "author_id": str(user_id), + "_type": "Comment", + "is_deleted": True, + } + return ForumComment()._collection.count_documents( + query_params + ) # pylint: disable=protected-access + + @classmethod + def restore_user_deleted_comments(cls, user_id, course_ids, restored_by=None): + """ + Restores (undeletes) comments of user in the given course_ids by setting is_deleted=False. + """ + return forum_api.restore_user_deleted_comments( + user_id=str(user_id), + course_ids=course_ids, + course_id=course_ids[0] if course_ids else None, + restored_by=restored_by, + ) + + @classmethod + def restore_comment(cls, comment_id, course_id=None, restored_by=None): + """ + Restores an individual soft-deleted comment by setting is_deleted=False + Public method for individual comment restoration + """ + return forum_api.restore_comment( + comment_id=comment_id, course_id=course_id, restored_by=restored_by + ) + def _url_for_thread_comments(thread_id): return f"{settings.PREFIX}/threads/{thread_id}/comments" diff --git a/openedx/core/djangoapps/django_comment_common/comment_client/models.py b/openedx/core/djangoapps/django_comment_common/comment_client/models.py index 4544a463ed80..ddfcc37cc524 100644 --- a/openedx/core/djangoapps/django_comment_common/comment_client/models.py +++ b/openedx/core/djangoapps/django_comment_common/comment_client/models.py @@ -4,24 +4,28 @@ import logging import typing as t -from .utils import CommentClientRequestError, extract, perform_request, get_course_key from forum import api as forum_api -from openedx.core.djangoapps.discussions.config.waffle import is_forum_v2_enabled, is_forum_v2_disabled_globally +from openedx.core.djangoapps.discussions.config.waffle import ( + is_forum_v2_disabled_globally, + is_forum_v2_enabled, +) + +from .utils import CommentClientRequestError, extract, get_course_key, perform_request log = logging.getLogger(__name__) class Model: - accessible_fields = ['id'] - updatable_fields = ['id'] - initializable_fields = ['id'] + accessible_fields = ["id"] + updatable_fields = ["id"] + initializable_fields = ["id"] base_url = None default_retrieve_params = {} metric_tag_fields = [] - DEFAULT_ACTIONS_WITH_ID = ['get', 'put', 'delete'] - DEFAULT_ACTIONS_WITHOUT_ID = ['get_all', 'post'] + DEFAULT_ACTIONS_WITH_ID = ["get", "put", "delete"] + DEFAULT_ACTIONS_WITHOUT_ID = ["get_all", "post"] DEFAULT_ACTIONS = DEFAULT_ACTIONS_WITH_ID + DEFAULT_ACTIONS_WITHOUT_ID def __init__(self, *args, **kwargs): @@ -29,18 +33,21 @@ def __init__(self, *args, **kwargs): self.retrieved = False def __getattr__(self, name): - if name == 'id': - return self.attributes.get('id', None) + if name == "id": + return self.attributes.get("id", None) try: return self.attributes[name] - except KeyError: + except KeyError as e: if self.retrieved or self.id is None: - raise AttributeError(f"Field {name} does not exist") # lint-amnesty, pylint: disable=raise-missing-from + raise AttributeError(f"Field {name} does not exist") from e self.retrieve() return self.__getattr__(name) def __setattr__(self, name, value): - if name == 'attributes' or name not in self.accessible_fields + self.updatable_fields: + if ( + name == "attributes" + or name not in self.accessible_fields + self.updatable_fields + ): super().__setattr__(name, value) else: self.attributes[name] = value @@ -76,7 +83,9 @@ def _retrieve(self, *args, **kwargs): if not course_id: _, course_id = is_forum_v2_enabled_for_comment(self.id) if self.type == "comment": - response = forum_api.get_parent_comment(comment_id=self.attributes["id"], course_id=course_id) + response = forum_api.get_parent_comment( + comment_id=self.attributes["id"], course_id=course_id + ) else: raise CommentClientRequestError("Forum v2 API call is missing") self._update_from_response(response) @@ -91,11 +100,11 @@ def _metric_tags(self): record the class name of the model. """ tags = [ - f'{self.__class__.__name__}.{attr}:{self[attr]}' + f"{self.__class__.__name__}.{attr}:{self[attr]}" for attr in self.metric_tag_fields if attr in self.attributes ] - tags.append(f'model_class:{self.__class__.__name__}') + tags.append(f"model_class:{self.__class__.__name__}") return tags @classmethod @@ -114,11 +123,11 @@ def retrieve_all(cls, params=None): The parsed JSON response from the backend. """ return perform_request( - 'get', - cls.url(action='get_all'), + "get", + cls.url(action="get_all"), params, - metric_tags=[f'model_class:{cls.__name__}'], - metric_action='model.retrieve_all', + metric_tags=[f"model_class:{cls.__name__}"], + metric_action="model.retrieve_all", ) def _update_from_response(self, response_data): @@ -128,8 +137,7 @@ def _update_from_response(self, response_data): else: log.warning( "Unexpected field {field_name} in model {model_name}".format( - field_name=k, - model_name=self.__class__.__name__ + field_name=k, model_name=self.__class__.__name__ ) ) @@ -152,7 +160,7 @@ def save(self, params=None): Invokes Forum's POST/PUT service to create/update thread """ self.before_save(self) - if self.id: # if we have id already, treat this as an update + if self.id: # if we have id already, treat this as an update response = self.handle_update(params) else: # otherwise, treat this as an insert response = self.handle_create(params) @@ -160,13 +168,25 @@ def save(self, params=None): self._update_from_response(response) self.after_save(self) - def delete(self, course_id=None): + def delete(self, course_id=None, deleted_by=None): course_key = get_course_key(self.attributes.get("course_id") or course_id) response = None if self.type == "comment": - response = forum_api.delete_comment(comment_id=self.attributes["id"], course_id=str(course_key)) + response = ( + forum_api.delete_comment( # pylint: disable=unexpected-keyword-arg + comment_id=self.attributes["id"], + course_id=str(course_key), + deleted_by=deleted_by, + ) + ) elif self.type == "thread": - response = forum_api.delete_thread(thread_id=self.attributes["id"], course_id=str(course_key)) + response = ( + forum_api.delete_thread( # pylint: disable=unexpected-keyword-arg + thread_id=self.attributes["id"], + course_id=str(course_key), + deleted_by=deleted_by, + ) + ) if response is None: raise CommentClientRequestError("Forum v2 API call is missing") self.retrieved = True @@ -176,7 +196,7 @@ def delete(self, course_id=None): def url_with_id(cls, params=None): if params is None: params = {} - return cls.base_url + '/' + str(params['id']) + return cls.base_url + "/" + str(params["id"]) @classmethod def url_without_id(cls, params=None): @@ -187,17 +207,21 @@ def url(cls, action, params=None): if params is None: params = {} if cls.base_url is None: - raise CommentClientRequestError("Must provide base_url when using default url function") - if action not in cls.DEFAULT_ACTIONS: # lint-amnesty, pylint: disable=no-else-raise + raise CommentClientRequestError( + "Must provide base_url when using default url function" + ) + if action not in cls.DEFAULT_ACTIONS: raise ValueError( f"Invalid action {action}. The supported action must be in {str(cls.DEFAULT_ACTIONS)}" ) - elif action in cls.DEFAULT_ACTIONS_WITH_ID: + if action in cls.DEFAULT_ACTIONS_WITH_ID: try: return cls.url_with_id(params) - except KeyError: - raise CommentClientRequestError(f"Cannot perform action {action} without id") # lint-amnesty, pylint: disable=raise-missing-from - else: # action must be in DEFAULT_ACTIONS_WITHOUT_ID now + except KeyError as e: + raise CommentClientRequestError( + f"Cannot perform action {action} without id" + ) from e + else: # action must be in DEFAULT_ACTIONS_WITHOUT_ID now return cls.url_without_id() def handle_update(self, params=None): @@ -306,8 +330,8 @@ def handle_create(self, params=None): try: return handlers[self.type](course_key) - except KeyError as exc: - raise CommentClientRequestError(f"Unsupported type: {self.type}") from exc + except KeyError as e: + raise CommentClientRequestError(f"Unsupported type: {self.type}") from e def handle_create_comment(self, course_id): request_data = self.initializable_attributes() @@ -319,8 +343,8 @@ def handle_create_comment(self, course_id): "anonymous": request_data.get("anonymous", False), "anonymous_to_peers": request_data.get("anonymous_to_peers", False), } - if 'endorsed' in request_data: - params['endorsed'] = request_data['endorsed'] + if "endorsed" in request_data: + params["endorsed"] = request_data["endorsed"] if parent_id := self.attributes.get("parent_id"): params["parent_comment_id"] = parent_id response = forum_api.create_child_comment(**params) diff --git a/openedx/core/djangoapps/django_comment_common/comment_client/thread.py b/openedx/core/djangoapps/django_comment_common/comment_client/thread.py index 34ccd7bf2ce6..754fe0065f00 100644 --- a/openedx/core/djangoapps/django_comment_common/comment_client/thread.py +++ b/openedx/core/djangoapps/django_comment_common/comment_client/thread.py @@ -5,50 +5,104 @@ import time import typing as t +from django.core.exceptions import ObjectDoesNotExist from eventtracking import tracker +from rest_framework.serializers import ValidationError -from django.core.exceptions import ObjectDoesNotExist from forum import api as forum_api -from forum.api.threads import prepare_thread_api_response -from forum.backend import get_backend -from forum.backends.mongodb.threads import CommentThread -from forum.utils import ForumV2RequestError -from rest_framework.serializers import ValidationError +from forum.api.threads import ( + prepare_thread_api_response, +) # pylint: disable=import-error +from forum.backend import get_backend # pylint: disable=import-error +from forum.backends.mongodb.threads import CommentThread # pylint: disable=import-error +from forum.utils import ForumV2RequestError # pylint: disable=import-error +from openedx.core.djangoapps.discussions.config.waffle import ( + is_forum_v2_disabled_globally, + is_forum_v2_enabled, +) -from openedx.core.djangoapps.discussions.config.waffle import is_forum_v2_enabled, is_forum_v2_disabled_globally from . import models, settings, utils - log = logging.getLogger(__name__) class Thread(models.Model): # accessible_fields can be set and retrieved on the model accessible_fields = [ - 'id', 'title', 'body', 'anonymous', 'anonymous_to_peers', 'course_id', - 'closed', 'tags', 'votes', 'commentable_id', 'username', 'user_id', - 'created_at', 'updated_at', 'comments_count', 'unread_comments_count', - 'at_position_list', 'children', 'type', 'highlighted_title', - 'highlighted_body', 'endorsed', 'read', 'group_id', 'group_name', 'pinned', - 'abuse_flaggers', 'resp_skip', 'resp_limit', 'resp_total', 'thread_type', - 'endorsed_responses', 'non_endorsed_responses', 'non_endorsed_resp_total', - 'context', 'last_activity_at', 'closed_by', 'close_reason_code', 'edit_history', - 'is_spam', 'ai_moderation_reason', 'abuse_flagged', + "id", + "title", + "body", + "anonymous", + "anonymous_to_peers", + "course_id", + "closed", + "tags", + "votes", + "commentable_id", + "username", + "user_id", + "created_at", + "updated_at", + "comments_count", + "unread_comments_count", + "at_position_list", + "children", + "type", + "highlighted_title", + "highlighted_body", + "endorsed", + "read", + "group_id", + "group_name", + "pinned", + "abuse_flaggers", + "resp_skip", + "resp_limit", + "resp_total", + "thread_type", + "endorsed_responses", + "non_endorsed_responses", + "non_endorsed_resp_total", + "context", + "last_activity_at", + "closed_by", + "close_reason_code", + "edit_history", + "is_spam", + "ai_moderation_reason", + "abuse_flagged", + "is_deleted", + "deleted_at", + "deleted_by", ] # updateable_fields are sent in PUT requests updatable_fields = [ - 'title', 'body', 'anonymous', 'anonymous_to_peers', 'course_id', 'read', - 'closed', 'user_id', 'commentable_id', 'group_id', 'group_name', 'pinned', 'thread_type', - 'close_reason_code', 'edit_reason_code', 'closing_user_id', 'editing_user_id', + "title", + "body", + "anonymous", + "anonymous_to_peers", + "course_id", + "read", + "closed", + "user_id", + "commentable_id", + "group_id", + "group_name", + "pinned", + "thread_type", + "close_reason_code", + "edit_reason_code", + "closing_user_id", + "editing_user_id", ] # initializable_fields are sent in POST requests - initializable_fields = updatable_fields + ['thread_type', 'context'] + initializable_fields = updatable_fields + ["thread_type", "context"] base_url = f"{settings.PREFIX}/threads" - default_retrieve_params = {'recursive': False} - type = 'thread' + default_retrieve_params = {"recursive": False} + type = "thread" @classmethod def search(cls, query_params): @@ -58,82 +112,83 @@ def search(cls, query_params): # with_responses=False internally in the comment service, so no additional # optimization is required. params = { - 'page': 1, - 'per_page': 20, - 'course_id': query_params['course_id'], + "page": 1, + "per_page": 20, + "course_id": query_params["course_id"], } - params.update( - utils.strip_blank(utils.strip_none(query_params)) - ) + params.update(utils.strip_blank(utils.strip_none(query_params))) # Convert user_id and author_id to strings if present - for field in ['user_id', 'author_id']: + for field in ["user_id", "author_id"]: if value := params.get(field): params[field] = str(value) # Handle commentable_ids/commentable_id conversion - if commentable_ids := params.get('commentable_ids'): - params['commentable_ids'] = commentable_ids.split(',') - elif commentable_id := params.get('commentable_id'): - params['commentable_ids'] = [commentable_id] - params.pop('commentable_id', None) - + if commentable_ids := params.get("commentable_ids"): + params["commentable_ids"] = commentable_ids.split(",") + elif commentable_id := params.get("commentable_id"): + params["commentable_ids"] = [commentable_id] + params.pop("commentable_id", None) + if query_params.get("show_deleted", False): + params["is_deleted"] = True params = utils.clean_forum_params(params) - if query_params.get('text'): # Handle group_ids/group_id conversion - if group_ids := params.get('group_ids'): - params['group_ids'] = [int(group_id) for group_id in group_ids.split(',')] - elif group_id := params.get('group_id'): - params['group_ids'] = [int(group_id)] - params.pop('group_id', None) + if query_params.get("text"): # Handle group_ids/group_id conversion + if group_ids := params.get("group_ids"): + params["group_ids"] = [ + int(group_id) for group_id in group_ids.split(",") + ] + elif group_id := params.get("group_id"): + params["group_ids"] = [int(group_id)] + params.pop("group_id", None) response = forum_api.search_threads(**params) else: response = forum_api.get_user_threads(**params) - if query_params.get('text'): - search_query = query_params['text'] - course_id = query_params['course_id'] - group_id = query_params['group_id'] if 'group_id' in query_params else None - requested_page = params['page'] - total_results = response.get('total_results') - corrected_text = response.get('corrected_text') + if query_params.get("text"): + search_query = query_params["text"] + course_id = query_params["course_id"] + group_id = query_params["group_id"] if "group_id" in query_params else None + requested_page = params["page"] + total_results = response.get("total_results") + corrected_text = response.get("corrected_text") # Record search result metric to allow search quality analysis. # course_id is already included in the context for the event tracker tracker.emit( - 'edx.forum.searched', + "edx.forum.searched", { - 'query': search_query, - 'search_type': 'Content', - 'corrected_text': corrected_text, - 'group_id': group_id, - 'page': requested_page, - 'total_results': total_results, - } + "query": search_query, + "search_type": "Content", + "corrected_text": corrected_text, + "group_id": group_id, + "page": requested_page, + "total_results": total_results, + }, ) log.info( 'forum_text_search query="{search_query}" corrected_text="{corrected_text}" course_id={course_id} ' - 'group_id={group_id} page={requested_page} total_results={total_results}'.format( + "group_id={group_id} page={requested_page} total_results={total_results}".format( search_query=search_query, corrected_text=corrected_text, course_id=course_id, group_id=group_id, requested_page=requested_page, - total_results=total_results + total_results=total_results, ) ) return utils.CommentClientPaginatedResult( - collection=response.get('collection', []), - page=response.get('page', 1), - num_pages=response.get('num_pages', 1), - thread_count=response.get('thread_count', 0), - corrected_text=response.get('corrected_text', None) + collection=response.get("collection", []), + page=response.get("page", 1), + num_pages=response.get("num_pages", 1), + thread_count=response.get("thread_count", 0), + corrected_text=response.get("corrected_text", None), ) @classmethod def url_for_threads(cls, params=None): - if params and params.get('commentable_id'): + if params and params.get("commentable_id"): return "{prefix}/{commentable_id}/threads".format( prefix=settings.PREFIX, - commentable_id=params['commentable_id'], + commentable_id=params["commentable_id"], ) else: return f"{settings.PREFIX}/threads" @@ -146,9 +201,9 @@ def url_for_search_threads(cls): def url(cls, action, params=None): if params is None: params = {} - if action in ['get_all', 'post']: + if action in ["get_all", "post"]: return cls.url_for_threads(params) - elif action == 'search': + elif action == "search": return cls.url_for_search_threads() else: return super().url(action, params) @@ -158,21 +213,23 @@ def url(cls, action, params=None): # that subclasses don't need to override for this. def _retrieve(self, *args, **kwargs): request_params = { - 'recursive': kwargs.get('recursive'), - 'with_responses': kwargs.get('with_responses', False), - 'user_id': kwargs.get('user_id'), - 'mark_as_read': kwargs.get('mark_as_read', True), - 'resp_skip': kwargs.get('response_skip'), - 'resp_limit': kwargs.get('response_limit'), - 'reverse_order': kwargs.get('reverse_order', False), - 'merge_question_type_responses': kwargs.get('merge_question_type_responses', False) + "recursive": kwargs.get("recursive"), + "with_responses": kwargs.get("with_responses", False), + "user_id": kwargs.get("user_id"), + "mark_as_read": kwargs.get("mark_as_read", True), + "resp_skip": kwargs.get("response_skip"), + "resp_limit": kwargs.get("response_limit"), + "reverse_order": kwargs.get("reverse_order", False), + "merge_question_type_responses": kwargs.get( + "merge_question_type_responses", False + ), } request_params = utils.clean_forum_params(request_params) course_id = kwargs.get("course_id") if not course_id: _, course_id = is_forum_v2_enabled_for_thread(self.id) - if user_id := request_params.get('user_id'): - request_params['user_id'] = str(user_id) + if user_id := request_params.get("user_id"): + request_params["user_id"] = str(user_id) response = forum_api.get_thread( thread_id=self.id, params=request_params, @@ -181,7 +238,7 @@ def _retrieve(self, *args, **kwargs): self._update_from_response(response) def flagAbuse(self, user, voteable, course_id=None): - if voteable.type != 'thread': + if voteable.type != "thread": raise utils.CommentClientRequestError("Can only flag threads") course_key = utils.get_course_key(self.attributes.get("course_id") or course_id) @@ -189,12 +246,12 @@ def flagAbuse(self, user, voteable, course_id=None): thread_id=voteable.id, action="flag", user_id=str(user.id), - course_id=str(course_key) + course_id=str(course_key), ) voteable._update_from_response(response) def unFlagAbuse(self, user, voteable, removeAll, course_id=None): - if voteable.type != 'thread': + if voteable.type != "thread": raise utils.CommentClientRequestError("Can only unflag threads") course_key = utils.get_course_key(self.attributes.get("course_id") or course_id) @@ -203,7 +260,7 @@ def unFlagAbuse(self, user, voteable, removeAll, course_id=None): action="unflag", user_id=user.id, update_all=bool(removeAll), - course_id=str(course_key) + course_id=str(course_key), ) voteable._update_from_response(response) @@ -211,18 +268,14 @@ def unFlagAbuse(self, user, voteable, removeAll, course_id=None): def pin(self, user, thread_id, course_id=None): course_key = utils.get_course_key(self.attributes.get("course_id") or course_id) response = forum_api.pin_thread( - user_id=user.id, - thread_id=thread_id, - course_id=str(course_key) + user_id=user.id, thread_id=thread_id, course_id=str(course_key) ) self._update_from_response(response) def un_pin(self, user, thread_id, course_id=None): course_key = utils.get_course_key(self.attributes.get("course_id") or course_id) response = forum_api.unpin_thread( - user_id=user.id, - thread_id=thread_id, - course_id=str(course_key) + user_id=user.id, thread_id=thread_id, course_id=str(course_key) ) self._update_from_response(response) @@ -235,12 +288,15 @@ def get_user_threads_count(cls, user_id, course_ids): query_params = { "course_id": {"$in": course_ids}, "author_id": str(user_id), - "_type": "CommentThread" + "is_deleted": {"$ne": True}, + "_type": "CommentThread", } - return CommentThread()._collection.count_documents(query_params) # pylint: disable=protected-access + return CommentThread()._collection.count_documents( + query_params + ) # pylint: disable=protected-access @classmethod - def _delete_thread(cls, thread_id, course_id=None): + def _delete_thread(cls, thread_id, course_id=None, deleted_by=None): """ Deletes a thread """ @@ -257,34 +313,53 @@ def _delete_thread(cls, thread_id, course_id=None): ) from exc start_time = time.perf_counter() - backend.delete_comments_of_a_thread(thread_id) - log.info(f"{prefix} Delete comments of thread {time.perf_counter() - start_time} sec") + # backend.delete_comments_of_a_thread(thread_id) + count_of_response_deleted, count_of_replies_deleted = ( + backend.soft_delete_comments_of_a_thread(thread_id, deleted_by) + ) + log.info( + f"{prefix} Delete comments of thread {time.perf_counter() - start_time} sec" + ) try: start_time = time.perf_counter() serialized_data = prepare_thread_api_response(thread, backend) - log.info(f"{prefix} Prepare response {time.perf_counter() - start_time} sec") + log.info( + f"{prefix} Prepare response {time.perf_counter() - start_time} sec" + ) except ValidationError as error: log.error(f"Validation error in get_thread: {error}") - raise ForumV2RequestError("Failed to prepare thread API response") from error + raise ForumV2RequestError( + "Failed to prepare thread API response" + ) from error start_time = time.perf_counter() backend.delete_subscriptions_of_a_thread(thread_id) - log.info(f"{prefix} Delete subscriptions {time.perf_counter() - start_time} sec") + log.info( + f"{prefix} Delete subscriptions {time.perf_counter() - start_time} sec" + ) start_time = time.perf_counter() - result = backend.delete_thread(thread_id) + # result = backend.delete_thread(thread_id) + result = backend.soft_delete_thread(thread_id, deleted_by) log.info(f"{prefix} Delete thread {time.perf_counter() - start_time} sec") if result and not (thread["anonymous"] or thread["anonymous_to_peers"]): start_time = time.perf_counter() backend.update_stats_for_course( - thread["author_id"], thread["course_id"], threads=-1 + thread["author_id"], + thread["course_id"], + threads=-1, + responses=-count_of_response_deleted, + replies=-count_of_replies_deleted, + deleted_threads=1, + deleted_responses=count_of_response_deleted, + deleted_replies=count_of_replies_deleted, ) log.info(f"{prefix} Update stats {time.perf_counter() - start_time} sec") return serialized_data @classmethod - def delete_user_threads(cls, user_id, course_ids): + def delete_user_threads(cls, user_id, course_ids, deleted_by=None): """ Deletes threads of user in the given course_ids. TODO: Add support for MySQL backend as well @@ -293,21 +368,65 @@ def delete_user_threads(cls, user_id, course_ids): query_params = { "course_id": {"$in": course_ids}, "author_id": str(user_id), + "is_deleted": {"$ne": True}, } threads_deleted = 0 threads = CommentThread().get_list(**query_params) - log.info(f"<> Fetched threads for user {user_id} in {time.time() - start_time} seconds") + log.info( + f"<> Fetched threads for user {user_id} in {time.time() - start_time} seconds" + ) for thread in threads: start_time = time.time() thread_id = thread.get("_id") course_id = thread.get("course_id") if thread_id: - cls._delete_thread(thread_id, course_id=course_id) + cls._delete_thread( + thread_id, course_id=course_id, deleted_by=deleted_by + ) threads_deleted += 1 - log.info(f"<> Deleted thread {thread_id} in {time.time() - start_time} seconds." - f" Thread Found: {thread_id is not None}") + log.info( + f"<> Deleted thread {thread_id} in {time.time() - start_time} seconds." + f" Thread Found: {thread_id is not None}" + ) return threads_deleted + @classmethod + def get_user_deleted_threads_count(cls, user_id, course_ids): + """ + Returns count of deleted threads for user in the given course_ids. + """ + query_params = { + "course_id": {"$in": course_ids}, + "author_id": str(user_id), + "_type": "CommentThread", + "is_deleted": True, + } + return CommentThread()._collection.count_documents( + query_params + ) # pylint: disable=protected-access + + @classmethod + def restore_user_deleted_threads(cls, user_id, course_ids, restored_by=None): + """ + Restores (undeletes) threads of user in the given course_ids by setting is_deleted=False. + """ + return forum_api.restore_user_deleted_threads( + user_id=str(user_id), + course_ids=course_ids, + course_id=course_ids[0] if course_ids else None, + restored_by=restored_by, + ) + + @classmethod + def restore_thread(cls, thread_id, course_id=None, restored_by=None): + """ + Restores an individual soft-deleted thread by setting is_deleted=False + Public method for individual thread restoration + """ + return forum_api.restore_thread( + thread_id=thread_id, course_id=course_id, restored_by=restored_by + ) + def _url_for_flag_abuse_thread(thread_id): return f"{settings.PREFIX}/threads/{thread_id}/abuse_flag" From 5af58a5b899ee77423ccbc979dd628c77d06ac57 Mon Sep 17 00:00:00 2001 From: Nathan Sprenkle Date: Tue, 6 Jan 2026 13:21:40 -0500 Subject: [PATCH 162/351] feat: shift progress calculation to backend (openedx#37399) (#69) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This commit migrates the data calculation logic for the GradeSummary table, which was previously in the frontend-app-learning. This commit also introduces a new visibility option for assignment scores: “Never show individual assessment results, but show overall assessment results after the due date.” With this option, learners cannot see question-level correctness or scores at any time. However, once the due date has passed, they can view their overall score in the total grades section on the Progress page. These two changes are coupled with each other because it compromises the integrity of this data to do the score hiding logic on the front end. The corresponding frontend PR is: openedx/frontend-app-learning#1797 Co-authored-by: Muhammad Anas <88967643+Anas12091101@users.noreply.github.com> --- .../js/show-correctness-editor.underscore | 7 + .../course_home_api/progress/api.py | 212 ++++++++++++++++++ .../course_home_api/progress/serializers.py | 17 ++ .../progress/tests/test_api.py | 109 ++++++++- .../progress/tests/test_views.py | 10 +- .../course_home_api/progress/views.py | 30 ++- lms/djangoapps/courseware/tests/test_views.py | 27 ++- lms/djangoapps/grades/subsection_grade.py | 9 +- lms/templates/courseware/progress.html | 9 +- xmodule/graders.py | 3 +- xmodule/tests/test_graders.py | 8 + 11 files changed, 429 insertions(+), 12 deletions(-) diff --git a/cms/templates/js/show-correctness-editor.underscore b/cms/templates/js/show-correctness-editor.underscore index 3db6c3c27a5c..1b0dd896747a 100644 --- a/cms/templates/js/show-correctness-editor.underscore +++ b/cms/templates/js/show-correctness-editor.underscore @@ -35,6 +35,13 @@ <% } %> <%- gettext('If the subsection does not have a due date, learners always see their scores when they submit answers to assessments.') %>

    + +

    + <%- gettext('Learners do not see question-level correctness or scores before or after the due date. However, once the due date passes, they can see their overall score for the subsection on the Progress page.') %> +

    diff --git a/lms/djangoapps/course_home_api/progress/api.py b/lms/djangoapps/course_home_api/progress/api.py index b2a8634c59f4..f89ecd3d2596 100644 --- a/lms/djangoapps/course_home_api/progress/api.py +++ b/lms/djangoapps/course_home_api/progress/api.py @@ -2,14 +2,226 @@ Python APIs exposed for the progress tracking functionality of the course home API. """ +from __future__ import annotations + from django.contrib.auth import get_user_model from opaque_keys.edx.keys import CourseKey +from openedx.core.lib.grade_utils import round_away_from_zero +from xmodule.graders import ShowCorrectness +from datetime import datetime, timezone from lms.djangoapps.courseware.courses import get_course_blocks_completion_summary +from dataclasses import dataclass, field User = get_user_model() +@dataclass +class _AssignmentBucket: + """Holds scores and visibility info for one assignment type. + + Attributes: + assignment_type: Full assignment type name from the grading policy (for example, "Homework"). + num_total: The total number of assignments expected to contribute to the grade before any + drop-lowest rules are applied. + last_grade_publish_date: The most recent date when grades for all assignments of assignment_type + are released and included in the final grade. + scores: Per-subsection fractional scores (each value is ``earned / possible`` and falls in + the range 0–1). While awaiting published content we pad the list with zero placeholders + so that its length always matches ``num_total`` until real scores replace them. + visibilities: Mirrors ``scores`` index-for-index and records whether each subsection's + correctness feedback is visible to the learner (``True``), hidden (``False``), or not + yet populated (``None`` when the entry is a placeholder). + included: Tracks whether each subsection currently counts toward the learner's grade as + determined by ``SubsectionGrade.show_grades``. Values follow the same convention as + ``visibilities`` (``True`` / ``False`` / ``None`` placeholders). + assignments_created: Count of real subsections inserted into the bucket so far. Once this + reaches ``num_total``, all placeholder entries have been replaced with actual data. + """ + assignment_type: str + num_total: int + last_grade_publish_date: datetime + scores: list[float] = field(default_factory=list) + visibilities: list[bool | None] = field(default_factory=list) + included: list[bool | None] = field(default_factory=list) + assignments_created: int = 0 + + @classmethod + def with_placeholders(cls, assignment_type: str, num_total: int, now: datetime): + """Create a bucket prefilled with placeholder (empty) entries.""" + return cls( + assignment_type=assignment_type, + num_total=num_total, + last_grade_publish_date=now, + scores=[0] * num_total, + visibilities=[None] * num_total, + included=[None] * num_total, + ) + + def add_subsection(self, score: float, is_visible: bool, is_included: bool): + """Add a subsection’s score and visibility, replacing a placeholder if space remains.""" + if self.assignments_created < self.num_total: + if self.scores: + self.scores.pop(0) + if self.visibilities: + self.visibilities.pop(0) + if self.included: + self.included.pop(0) + self.scores.append(score) + self.visibilities.append(is_visible) + self.included.append(is_included) + self.assignments_created += 1 + + def drop_lowest(self, num_droppable: int): + """Remove the lowest scoring subsections, up to the provided num_droppable.""" + while num_droppable > 0 and self.scores: + idx = self.scores.index(min(self.scores)) + self.scores.pop(idx) + self.visibilities.pop(idx) + self.included.pop(idx) + num_droppable -= 1 + + def hidden_state(self) -> str: + """Return whether kept scores are all, some, or none hidden.""" + if not self.visibilities: + return 'none' + all_hidden = all(v is False for v in self.visibilities) + some_hidden = any(v is False for v in self.visibilities) + if all_hidden: + return 'all' + if some_hidden: + return 'some' + return 'none' + + def averages(self) -> tuple[float, float]: + """Compute visible and included averages over kept scores. + + Visible average uses only grades with visibility flag True in numerator; denominator is total + number of kept scores (mirrors legacy behavior). Included average uses only scores that are + marked included (show_grades True) in numerator with same denominator. + + Returns: + (earned_visible, earned_all) tuple of floats (0-1 each). + """ + if not self.scores: + return 0.0, 0.0 + visible_scores = [s for i, s in enumerate(self.scores) if self.visibilities[i]] + included_scores = [s for i, s in enumerate(self.scores) if self.included[i]] + earned_visible = (sum(visible_scores) / len(self.scores)) if self.scores else 0.0 + earned_all = (sum(included_scores) / len(self.scores)) if self.scores else 0.0 + return earned_visible, earned_all + + +class _AssignmentTypeGradeAggregator: + """Collects and aggregates subsection grades by assignment type.""" + + def __init__(self, course_grade, grading_policy: dict, has_staff_access: bool): + """Initialize with course grades, grading policy, and staff access flag.""" + self.course_grade = course_grade + self.grading_policy = grading_policy + self.has_staff_access = has_staff_access + self.now = datetime.now(timezone.utc) + self.policy_map = self._build_policy_map() + self.buckets: dict[str, _AssignmentBucket] = {} + + def _build_policy_map(self) -> dict: + """Convert grading policy into a lookup of assignment type → policy info.""" + policy_map = {} + for policy in self.grading_policy.get('GRADER', []): + policy_map[policy.get('type')] = { + 'weight': policy.get('weight', 0.0), + 'short_label': policy.get('short_label', ''), + 'num_droppable': policy.get('drop_count', 0), + 'num_total': policy.get('min_count', 0), + } + return policy_map + + def _bucket_for(self, assignment_type: str) -> _AssignmentBucket: + """Get or create a score bucket for the given assignment type.""" + bucket = self.buckets.get(assignment_type) + if bucket is None: + num_total = self.policy_map.get(assignment_type, {}).get('num_total', 0) or 0 + bucket = _AssignmentBucket.with_placeholders(assignment_type, num_total, self.now) + self.buckets[assignment_type] = bucket + return bucket + + def collect(self): + """Gather subsection grades into their respective assignment buckets.""" + for chapter in self.course_grade.chapter_grades.values(): + for subsection_grade in chapter.get('sections', []): + if not getattr(subsection_grade, 'graded', False): + continue + assignment_type = getattr(subsection_grade, 'format', '') or '' + if not assignment_type: + continue + graded_total = getattr(subsection_grade, 'graded_total', None) + earned = getattr(graded_total, 'earned', 0.0) if graded_total else 0.0 + possible = getattr(graded_total, 'possible', 0.0) if graded_total else 0.0 + earned = 0.0 if earned is None else earned + possible = 0.0 if possible is None else possible + score = (earned / possible) if possible else 0.0 + is_visible = ShowCorrectness.correctness_available( + subsection_grade.show_correctness, subsection_grade.due, self.has_staff_access + ) + is_included = subsection_grade.show_grades(self.has_staff_access) + bucket = self._bucket_for(assignment_type) + bucket.add_subsection(score, is_visible, is_included) + visibilities_with_due_dates = [ShowCorrectness.PAST_DUE, ShowCorrectness.NEVER_BUT_INCLUDE_GRADE] + if subsection_grade.show_correctness in visibilities_with_due_dates: + if subsection_grade.due and subsection_grade.due > bucket.last_grade_publish_date: + bucket.last_grade_publish_date = subsection_grade.due + + def build_results(self) -> dict: + """Apply drops, compute averages, and return aggregated results and total grade.""" + final_grades = 0.0 + rows = [] + for assignment_type, bucket in self.buckets.items(): + policy = self.policy_map.get(assignment_type, {}) + bucket.drop_lowest(policy.get('num_droppable', 0)) + earned_visible, earned_all = bucket.averages() + weight = policy.get('weight', 0.0) + short_label = policy.get('short_label', '') + row = { + 'type': assignment_type, + 'weight': weight, + 'average_grade': round_away_from_zero(earned_visible, 4), + 'weighted_grade': round_away_from_zero(earned_visible * weight, 4), + 'short_label': short_label, + 'num_droppable': policy.get('num_droppable', 0), + 'last_grade_publish_date': bucket.last_grade_publish_date, + 'has_hidden_contribution': bucket.hidden_state(), + } + final_grades += earned_all * weight + rows.append(row) + rows.sort(key=lambda r: r['weight']) + return {'results': rows, 'final_grades': round_away_from_zero(final_grades, 4)} + + def run(self) -> dict: + """Execute full pipeline (collect + aggregate) returning final payload.""" + self.collect() + return self.build_results() + + +def aggregate_assignment_type_grade_summary( + course_grade, + grading_policy: dict, + has_staff_access: bool = False, +) -> dict: + """ + Aggregate subsection grades by assignment type and return summary data. + Args: + course_grade: CourseGrade object containing chapter and subsection grades. + grading_policy: Dictionary representing the course's grading policy. + has_staff_access: Boolean indicating if the user has staff access to view all grades. + Returns: + Dictionary with keys: + results: list of per-assignment-type summary dicts + final_grades: overall weighted contribution (float, 4 decimal rounding) + """ + aggregator = _AssignmentTypeGradeAggregator(course_grade, grading_policy, has_staff_access) + return aggregator.run() + + def calculate_progress_for_learner_in_course(course_key: CourseKey, user: User) -> dict: """ Calculate a given learner's progress in the specified course run. diff --git a/lms/djangoapps/course_home_api/progress/serializers.py b/lms/djangoapps/course_home_api/progress/serializers.py index 6bdc204434af..c48660a41c6a 100644 --- a/lms/djangoapps/course_home_api/progress/serializers.py +++ b/lms/djangoapps/course_home_api/progress/serializers.py @@ -26,6 +26,7 @@ class SubsectionScoresSerializer(ReadOnlySerializer): assignment_type = serializers.CharField(source='format') block_key = serializers.SerializerMethodField() display_name = serializers.CharField() + due = serializers.DateTimeField(allow_null=True) has_graded_assignment = serializers.BooleanField(source='graded') override = serializers.SerializerMethodField() learner_has_access = serializers.SerializerMethodField() @@ -127,6 +128,20 @@ class VerificationDataSerializer(ReadOnlySerializer): status_date = serializers.DateTimeField() +class AssignmentTypeScoresSerializer(ReadOnlySerializer): + """ + Serializer for aggregated scores per assignment type. + """ + type = serializers.CharField() + weight = serializers.FloatField() + average_grade = serializers.FloatField() + weighted_grade = serializers.FloatField() + last_grade_publish_date = serializers.DateTimeField() + has_hidden_contribution = serializers.CharField() + short_label = serializers.CharField() + num_droppable = serializers.IntegerField() + + class ProgressTabSerializer(VerifiedModeSerializer): """ Serializer for progress tab @@ -146,3 +161,5 @@ class ProgressTabSerializer(VerifiedModeSerializer): user_has_passing_grade = serializers.BooleanField() verification_data = VerificationDataSerializer() disable_progress_graph = serializers.BooleanField() + assignment_type_grade_summary = AssignmentTypeScoresSerializer(many=True) + final_grades = serializers.FloatField() diff --git a/lms/djangoapps/course_home_api/progress/tests/test_api.py b/lms/djangoapps/course_home_api/progress/tests/test_api.py index 30d8d9059eaa..51e7dd68286e 100644 --- a/lms/djangoapps/course_home_api/progress/tests/test_api.py +++ b/lms/djangoapps/course_home_api/progress/tests/test_api.py @@ -6,7 +6,80 @@ from django.test import TestCase -from lms.djangoapps.course_home_api.progress.api import calculate_progress_for_learner_in_course +from lms.djangoapps.course_home_api.progress.api import ( + calculate_progress_for_learner_in_course, + aggregate_assignment_type_grade_summary, +) +from xmodule.graders import ShowCorrectness +from datetime import datetime, timedelta, timezone +from types import SimpleNamespace + + +def _make_subsection(fmt, earned, possible, show_corr, *, due_delta_days=None): + """Build a lightweight subsection object for testing aggregation scenarios.""" + graded_total = SimpleNamespace(earned=earned, possible=possible) + due = None + if due_delta_days is not None: + due = datetime.now(timezone.utc) + timedelta(days=due_delta_days) + return SimpleNamespace( + graded=True, + format=fmt, + graded_total=graded_total, + show_correctness=show_corr, + due=due, + show_grades=lambda staff: True, + ) + + +_AGGREGATION_SCENARIOS = [ + ( + 'all_visible_always', + {'type': 'Homework', 'weight': 1.0, 'drop_count': 0, 'min_count': 2, 'short_label': 'HW'}, + [ + _make_subsection('Homework', 1, 1, ShowCorrectness.ALWAYS), + _make_subsection('Homework', 0.5, 1, ShowCorrectness.ALWAYS), + ], + {'avg': 0.75, 'weighted': 0.75, 'hidden': 'none', 'final': 0.75}, + ), + ( + 'some_hidden_never_but_include', + {'type': 'Exam', 'weight': 1.0, 'drop_count': 0, 'min_count': 2, 'short_label': 'EX'}, + [ + _make_subsection('Exam', 1, 1, ShowCorrectness.ALWAYS), + _make_subsection('Exam', 0.5, 1, ShowCorrectness.NEVER_BUT_INCLUDE_GRADE), + ], + {'avg': 0.5, 'weighted': 0.5, 'hidden': 'some', 'final': 0.75}, + ), + ( + 'all_hidden_never_but_include', + {'type': 'Quiz', 'weight': 1.0, 'drop_count': 0, 'min_count': 2, 'short_label': 'QZ'}, + [ + _make_subsection('Quiz', 0.4, 1, ShowCorrectness.NEVER_BUT_INCLUDE_GRADE), + _make_subsection('Quiz', 0.6, 1, ShowCorrectness.NEVER_BUT_INCLUDE_GRADE), + ], + {'avg': 0.0, 'weighted': 0.0, 'hidden': 'all', 'final': 0.5}, + ), + ( + 'past_due_mixed_visibility', + {'type': 'Lab', 'weight': 1.0, 'drop_count': 0, 'min_count': 2, 'short_label': 'LB'}, + [ + _make_subsection('Lab', 0.8, 1, ShowCorrectness.PAST_DUE, due_delta_days=-1), + _make_subsection('Lab', 0.2, 1, ShowCorrectness.PAST_DUE, due_delta_days=+3), + ], + {'avg': 0.4, 'weighted': 0.4, 'hidden': 'some', 'final': 0.5}, + ), + ( + 'drop_lowest_keeps_high_scores', + {'type': 'Project', 'weight': 1.0, 'drop_count': 2, 'min_count': 4, 'short_label': 'PR'}, + [ + _make_subsection('Project', 1, 1, ShowCorrectness.ALWAYS), + _make_subsection('Project', 1, 1, ShowCorrectness.ALWAYS), + _make_subsection('Project', 0, 1, ShowCorrectness.ALWAYS), + _make_subsection('Project', 0, 1, ShowCorrectness.ALWAYS), + ], + {'avg': 1.0, 'weighted': 1.0, 'hidden': 'none', 'final': 1.0}, + ), +] class ProgressApiTests(TestCase): @@ -73,3 +146,37 @@ def test_calculate_progress_for_learner_in_course_summary_empty(self, mock_get_s results = calculate_progress_for_learner_in_course("some_course", "some_user") assert not results + + def test_aggregate_assignment_type_grade_summary_scenarios(self): + """ + A test to verify functionality of aggregate_assignment_type_grade_summary. + 1. Test visibility modes (always, never but include grade, past due) + 2. Test drop-lowest behavior + 3. Test weighting behavior + 4. Test final grade calculation + 5. Test average grade calculation + 6. Test weighted grade calculation + 7. Test has_hidden_contribution calculation + """ + + for case_name, policy, subsections, expected in _AGGREGATION_SCENARIOS: + with self.subTest(case_name=case_name): + course_grade = SimpleNamespace(chapter_grades={'chapter': {'sections': subsections}}) + grading_policy = {'GRADER': [policy]} + + result = aggregate_assignment_type_grade_summary( + course_grade, + grading_policy, + has_staff_access=False, + ) + + assert 'results' in result and 'final_grades' in result + assert result['final_grades'] == expected['final'] + assert len(result['results']) == 1 + + row = result['results'][0] + assert row['type'] == policy['type'], case_name + assert row['average_grade'] == expected['avg'] + assert row['weighted_grade'] == expected['weighted'] + assert row['has_hidden_contribution'] == expected['hidden'] + assert row['num_droppable'] == policy['drop_count'] diff --git a/lms/djangoapps/course_home_api/progress/tests/test_views.py b/lms/djangoapps/course_home_api/progress/tests/test_views.py index d13ebec29c21..8012e11675f1 100644 --- a/lms/djangoapps/course_home_api/progress/tests/test_views.py +++ b/lms/djangoapps/course_home_api/progress/tests/test_views.py @@ -282,8 +282,8 @@ def test_url_hidden_if_subsection_hide_after_due(self): assert hide_after_due_subsection['url'] is None @ddt.data( - (True, 0.7), # midterm and final are visible to staff - (False, 0.3), # just the midterm is visible to learners + (True, 0.72), # lab, midterm and final are visible to staff + (False, 0.32), # Only lab and midterm is visible to learners ) @ddt.unpack def test_course_grade_considers_subsection_grade_visibility(self, is_staff, expected_percent): @@ -301,14 +301,18 @@ def test_course_grade_considers_subsection_grade_visibility(self, is_staff, expe never = self.add_subsection_with_problem(format='Homework', show_correctness='never') always = self.add_subsection_with_problem(format='Midterm Exam', show_correctness='always') past_due = self.add_subsection_with_problem(format='Final Exam', show_correctness='past_due', due=tomorrow) + never_but_show_grade = self.add_subsection_with_problem( + format='Lab', show_correctness='never_but_include_grade' + ) answer_problem(self.course, get_mock_request(self.user), never) answer_problem(self.course, get_mock_request(self.user), always) answer_problem(self.course, get_mock_request(self.user), past_due) + answer_problem(self.course, get_mock_request(self.user), never_but_show_grade) # First, confirm the grade in the database - it should never change based on user state. # This is midterm and final and a single problem added together. - assert CourseGradeFactory().read(self.user, self.course).percent == 0.72 + assert CourseGradeFactory().read(self.user, self.course).percent == 0.73 response = self.client.get(self.url) assert response.status_code == 200 diff --git a/lms/djangoapps/course_home_api/progress/views.py b/lms/djangoapps/course_home_api/progress/views.py index 3783c19061dc..54e71df48cc5 100644 --- a/lms/djangoapps/course_home_api/progress/views.py +++ b/lms/djangoapps/course_home_api/progress/views.py @@ -13,8 +13,11 @@ from rest_framework.response import Response from xmodule.modulestore.django import modulestore +from xmodule.graders import ShowCorrectness from common.djangoapps.student.models import CourseEnrollment from lms.djangoapps.course_home_api.progress.serializers import ProgressTabSerializer +from lms.djangoapps.course_home_api.progress.api import aggregate_assignment_type_grade_summary + from lms.djangoapps.course_home_api.toggles import course_home_mfe_progress_tab_is_active from lms.djangoapps.courseware.access import has_access, has_ccx_coach_role from lms.djangoapps.course_blocks.api import get_course_blocks @@ -99,6 +102,7 @@ class ProgressTabView(RetrieveAPIView): assignment_type: (str) the format, if any, of the Subsection (Homework, Exam, etc) block_key: (str) the key of the given subsection block display_name: (str) a str of what the name of the Subsection is for displaying on the site + due: (str or None) the due date of the subsection in ISO 8601 format, or None if no due date is set has_graded_assignment: (bool) whether or not the Subsection is a graded assignment learner_has_access: (bool) whether the learner has access to the subsection (could be FBE gated) num_points_earned: (int) the amount of points the user has earned for the given subsection @@ -175,6 +179,18 @@ def _get_student_user(self, request, course_key, student_id, is_staff): except User.DoesNotExist as exc: raise Http404 from exc + def _visible_section_scores(self, course_grade): + """Return only those chapter/section scores that are visible to the learner.""" + visible_chapters = [] + for chapter in course_grade.chapter_grades.values(): + filtered_sections = [ + subsection + for subsection in chapter["sections"] + if getattr(subsection, "show_correctness", None) != ShowCorrectness.NEVER_BUT_INCLUDE_GRADE + ] + visible_chapters.append({**chapter, "sections": filtered_sections}) + return visible_chapters + def get(self, request, *args, **kwargs): course_key_string = kwargs.get('course_key_string') course_key = CourseKey.from_string(course_key_string) @@ -245,6 +261,16 @@ def get(self, request, *args, **kwargs): access_expiration = get_access_expiration_data(request.user, course_overview) + # Aggregations delegated to helper functions for reuse and testability + assignment_type_grade_summary = aggregate_assignment_type_grade_summary( + course_grade, + grading_policy, + has_staff_access=is_staff, + ) + + # Filter out section scores to only have those that are visible to the user + section_scores = self._visible_section_scores(course_grade) + data = { 'access_expiration': access_expiration, 'certificate_data': get_cert_data(student, course, enrollment_mode, course_grade), @@ -255,12 +281,14 @@ def get(self, request, *args, **kwargs): 'enrollment_mode': enrollment_mode, 'grading_policy': grading_policy, 'has_scheduled_content': has_scheduled_content, - 'section_scores': list(course_grade.chapter_grades.values()), + 'section_scores': section_scores, 'studio_url': get_studio_url(course, 'settings/grading'), 'username': username, 'user_has_passing_grade': user_has_passing_grade, 'verification_data': verification_data, 'disable_progress_graph': disable_progress_graph, + 'assignment_type_grade_summary': assignment_type_grade_summary["results"], + 'final_grades': assignment_type_grade_summary["final_grades"], } context = self.get_serializer_context() context['staff_access'] = is_staff diff --git a/lms/djangoapps/courseware/tests/test_views.py b/lms/djangoapps/courseware/tests/test_views.py index 4e3d1be9bddc..2c3ece3133a5 100644 --- a/lms/djangoapps/courseware/tests/test_views.py +++ b/lms/djangoapps/courseware/tests/test_views.py @@ -1781,6 +1781,14 @@ def assert_progress_page_show_grades(self, response, show_correctness, due_date, (ShowCorrectness.PAST_DUE, TODAY, True), (ShowCorrectness.PAST_DUE, TOMORROW, False), (ShowCorrectness.PAST_DUE, TOMORROW, True), + (ShowCorrectness.NEVER_BUT_INCLUDE_GRADE, None, False), + (ShowCorrectness.NEVER_BUT_INCLUDE_GRADE, None, True), + (ShowCorrectness.NEVER_BUT_INCLUDE_GRADE, YESTERDAY, False), + (ShowCorrectness.NEVER_BUT_INCLUDE_GRADE, YESTERDAY, True), + (ShowCorrectness.NEVER_BUT_INCLUDE_GRADE, TODAY, False), + (ShowCorrectness.NEVER_BUT_INCLUDE_GRADE, TODAY, True), + (ShowCorrectness.NEVER_BUT_INCLUDE_GRADE, TOMORROW, False), + (ShowCorrectness.NEVER_BUT_INCLUDE_GRADE, TOMORROW, True), ) @ddt.unpack def test_progress_page_no_problem_scores(self, show_correctness, due_date_name, graded): @@ -1821,6 +1829,14 @@ def test_progress_page_no_problem_scores(self, show_correctness, due_date_name, (ShowCorrectness.PAST_DUE, TODAY, True, True), (ShowCorrectness.PAST_DUE, TOMORROW, False, False), (ShowCorrectness.PAST_DUE, TOMORROW, True, False), + (ShowCorrectness.NEVER_BUT_INCLUDE_GRADE, None, False, False), + (ShowCorrectness.NEVER_BUT_INCLUDE_GRADE, None, True, False), + (ShowCorrectness.NEVER_BUT_INCLUDE_GRADE, YESTERDAY, False, False), + (ShowCorrectness.NEVER_BUT_INCLUDE_GRADE, YESTERDAY, True, False), + (ShowCorrectness.NEVER_BUT_INCLUDE_GRADE, TODAY, False, False), + (ShowCorrectness.NEVER_BUT_INCLUDE_GRADE, TODAY, True, False), + (ShowCorrectness.NEVER_BUT_INCLUDE_GRADE, TOMORROW, False, False), + (ShowCorrectness.NEVER_BUT_INCLUDE_GRADE, TOMORROW, True, False), ) @ddt.unpack def test_progress_page_hide_scores_from_learner(self, show_correctness, due_date_name, graded, show_grades): @@ -1873,11 +1889,20 @@ def test_progress_page_hide_scores_from_learner(self, show_correctness, due_date (ShowCorrectness.PAST_DUE, TODAY, True, True), (ShowCorrectness.PAST_DUE, TOMORROW, False, True), (ShowCorrectness.PAST_DUE, TOMORROW, True, True), + (ShowCorrectness.NEVER_BUT_INCLUDE_GRADE, None, False, False), + (ShowCorrectness.NEVER_BUT_INCLUDE_GRADE, None, True, False), + (ShowCorrectness.NEVER_BUT_INCLUDE_GRADE, YESTERDAY, False, False), + (ShowCorrectness.NEVER_BUT_INCLUDE_GRADE, YESTERDAY, True, False), + (ShowCorrectness.NEVER_BUT_INCLUDE_GRADE, TODAY, False, False), + (ShowCorrectness.NEVER_BUT_INCLUDE_GRADE, TODAY, True, False), + (ShowCorrectness.NEVER_BUT_INCLUDE_GRADE, TOMORROW, False, False), + (ShowCorrectness.NEVER_BUT_INCLUDE_GRADE, TOMORROW, True, False), ) @ddt.unpack def test_progress_page_hide_scores_from_staff(self, show_correctness, due_date_name, graded, show_grades): """ - Test that problem scores are hidden from staff viewing a learner's progress page only if show_correctness=never. + Test that problem scores are hidden from staff viewing a learner's progress page only if show_correctness is + never or never_but_include_grade. """ due_date = self.DATES[due_date_name] self.setup_course(show_correctness=show_correctness, due_date=due_date, graded=graded) diff --git a/lms/djangoapps/grades/subsection_grade.py b/lms/djangoapps/grades/subsection_grade.py index 4ce0a1f3a463..b0c98497b823 100644 --- a/lms/djangoapps/grades/subsection_grade.py +++ b/lms/djangoapps/grades/subsection_grade.py @@ -5,8 +5,8 @@ from abc import ABCMeta from collections import OrderedDict +from datetime import datetime, timezone from logging import getLogger - from lazy import lazy from lms.djangoapps.grades.models import BlockRecord, PersistentSubsectionGrade @@ -59,6 +59,13 @@ def show_grades(self, has_staff_access): """ Returns whether subsection scores are currently available to users with or without staff access. """ + if self.show_correctness == ShowCorrectness.NEVER_BUT_INCLUDE_GRADE: + # show_grades fn is used to determine if the grade should be included in final calculation. + # For NEVER_BUT_INCLUDE_GRADE, show_grades returns True if the due date has passed, + # but correctness_available always returns False as we do not want to show correctness + # of problems to the users. + return (self.due is None or + self.due < datetime.now(timezone.utc)) return ShowCorrectness.correctness_available(self.show_correctness, self.due, has_staff_access) @property diff --git a/lms/templates/courseware/progress.html b/lms/templates/courseware/progress.html index 711fad895427..3ee4044fcbbf 100644 --- a/lms/templates/courseware/progress.html +++ b/lms/templates/courseware/progress.html @@ -16,6 +16,7 @@ from lms.djangoapps.grades.api import constants as grades_constants from openedx.core.djangolib.markup import HTML, Text from openedx.features.enterprise_support.utils import get_enterprise_learner_generic_name +from xmodule.graders import ShowCorrectness %> <% @@ -180,7 +181,7 @@

    ${ chapter['display_name']}

    %if hide_url:

    ${section.display_name} - %if (total > 0 or earned > 0) and section.show_grades(staff_access): + %if (total > 0 or earned > 0) and ShowCorrectness.correctness_available(section.show_correctness, section.due, staff_access): ${_("{earned} of {total} possible points").format(earned='{:.3n}'.format(float(earned)), total='{:.3n}'.format(float(total)))} @@ -189,14 +190,14 @@

    %else: ${ section.display_name} - %if (total > 0 or earned > 0) and section.show_grades(staff_access): + %if (total > 0 or earned > 0) and ShowCorrectness.correctness_available(section.show_correctness, section.due, staff_access): ${_("{earned} of {total} possible points").format(earned='{:.3n}'.format(float(earned)), total='{:.3n}'.format(float(total)))} %endif %endif - %if (total > 0 or earned > 0) and section.show_grades(staff_access): + %if (total > 0 or earned > 0) and ShowCorrectness.correctness_available(section.show_correctness, section.due, staff_access): ${"({0:.3n}/{1:.3n}) {2}".format( float(earned), float(total), percentageString )} %endif

    @@ -219,7 +220,7 @@

    %endif

    %if len(section.problem_scores.values()) > 0: - %if section.show_grades(staff_access): + %if ShowCorrectness.correctness_available(section.show_correctness, section.due, staff_access):
    ${ _("Problem Scores: ") if section.graded else _("Practice Scores: ")}
    %for score in section.problem_scores.values(): diff --git a/xmodule/graders.py b/xmodule/graders.py index 001113882438..34f7e61654b4 100644 --- a/xmodule/graders.py +++ b/xmodule/graders.py @@ -485,13 +485,14 @@ class ShowCorrectness: ALWAYS = "always" PAST_DUE = "past_due" NEVER = "never" + NEVER_BUT_INCLUDE_GRADE = "never_but_include_grade" @classmethod def correctness_available(cls, show_correctness='', due_date=None, has_staff_access=False): """ Returns whether correctness is available now, for the given attributes. """ - if show_correctness == cls.NEVER: + if show_correctness in (cls.NEVER, cls.NEVER_BUT_INCLUDE_GRADE): return False elif has_staff_access: # This is after the 'never' check because course staff can see correctness diff --git a/xmodule/tests/test_graders.py b/xmodule/tests/test_graders.py index 5e2444a05533..b80004c913a3 100644 --- a/xmodule/tests/test_graders.py +++ b/xmodule/tests/test_graders.py @@ -493,3 +493,11 @@ def test_show_correctness_past_due(self, due_date_str, has_staff_access, expecte due_date = getattr(self, due_date_str) assert ShowCorrectness.correctness_available(ShowCorrectness.PAST_DUE, due_date, has_staff_access) ==\ expected_result + + @ddt.data(True, False) + def test_show_correctness_never_but_include_grade(self, has_staff_access): + """ + Test that show_correctness="never_but_include_grade" hides correctness from learners and course staff. + """ + assert not ShowCorrectness.correctness_available(show_correctness=ShowCorrectness.NEVER_BUT_INCLUDE_GRADE, + has_staff_access=has_staff_access) From e434680d958e443e983e16c5a8c1a1ce4082cadc Mon Sep 17 00:00:00 2001 From: Chintan Joshi Date: Wed, 7 Jan 2026 21:37:47 +0530 Subject: [PATCH 163/351] Revert "feat: soft delete (#37)" (#78) This reverts commit 1e8714f6c0b27f3bfc50ab33a317bc860846d978. --- lms/djangoapps/discussion/rest_api/api.py | 1272 ++++------------ lms/djangoapps/discussion/rest_api/forms.py | 73 +- .../discussion/rest_api/serializers.py | 287 +--- lms/djangoapps/discussion/rest_api/tasks.py | 110 +- .../discussion/rest_api/tests/test_api_v2.py | 105 +- .../discussion/rest_api/tests/test_forms.py | 99 +- .../rest_api/tests/test_serializers.py | 592 ++++---- .../discussion/rest_api/tests/test_views.py | 1284 +++++++---------- .../rest_api/tests/test_views_v2.py | 65 +- .../discussion/rest_api/tests/utils.py | 440 +++--- lms/djangoapps/discussion/rest_api/urls.py | 64 +- lms/djangoapps/discussion/rest_api/views.py | 583 ++------ .../comment_client/comment.py | 142 +- .../comment_client/models.py | 96 +- .../comment_client/thread.py | 325 ++--- 15 files changed, 1800 insertions(+), 3737 deletions(-) diff --git a/lms/djangoapps/discussion/rest_api/api.py b/lms/djangoapps/discussion/rest_api/api.py index fcc13efc40b8..b87852c16cfa 100644 --- a/lms/djangoapps/discussion/rest_api/api.py +++ b/lms/djangoapps/discussion/rest_api/api.py @@ -1,17 +1,17 @@ """ Discussion API internal interface """ - from __future__ import annotations import itertools -import logging import re from collections import defaultdict from datetime import datetime + from enum import Enum from typing import Dict, Iterable, List, Literal, Optional, Set, Tuple from urllib.parse import urlencode, urlunparse +from pytz import UTC from django.conf import settings from django.contrib.auth import get_user_model @@ -19,26 +19,24 @@ from django.db.models import Q from django.http import Http404 from django.urls import reverse -from django.utils.html import strip_tags from edx_django_utils.monitoring import function_trace from opaque_keys import InvalidKeyError from opaque_keys.edx.locator import CourseKey -from pytz import UTC from rest_framework import status from rest_framework.exceptions import PermissionDenied from rest_framework.request import Request from rest_framework.response import Response -from common.djangoapps.student.roles import CourseInstructorRole, CourseStaffRole -from forum import api as forum_api +from common.djangoapps.student.roles import ( + CourseInstructorRole, + CourseStaffRole, +) + from lms.djangoapps.course_api.blocks.api import get_blocks from lms.djangoapps.courseware.courses import get_course_with_access from lms.djangoapps.courseware.exceptions import CourseAccessRedirect from lms.djangoapps.discussion.rate_limit import is_content_creation_rate_limited -from lms.djangoapps.discussion.toggles import ( - ENABLE_DISCUSSIONS_MFE, - ONLY_VERIFIED_USERS_CAN_POST, -) +from lms.djangoapps.discussion.toggles import ENABLE_DISCUSSIONS_MFE, ONLY_VERIFIED_USERS_CAN_POST from lms.djangoapps.discussion.views import is_privileged_user from openedx.core.djangoapps.discussions.models import ( DiscussionsConfiguration, @@ -50,12 +48,12 @@ from openedx.core.djangoapps.django_comment_common.comment_client.comment import Comment from openedx.core.djangoapps.django_comment_common.comment_client.course import ( get_course_commentable_counts, - get_course_user_stats, + get_course_user_stats ) from openedx.core.djangoapps.django_comment_common.comment_client.thread import Thread from openedx.core.djangoapps.django_comment_common.comment_client.utils import ( CommentClient500Error, - CommentClientRequestError, + CommentClientRequestError ) from openedx.core.djangoapps.django_comment_common.models import ( FORUM_ROLE_ADMINISTRATOR, @@ -63,13 +61,13 @@ FORUM_ROLE_GROUP_MODERATOR, FORUM_ROLE_MODERATOR, CourseDiscussionSettings, - Role, + Role ) from openedx.core.djangoapps.django_comment_common.signals import ( comment_created, comment_deleted, - comment_edited, comment_endorsed, + comment_edited, comment_flagged, comment_voted, thread_created, @@ -77,15 +75,11 @@ thread_edited, thread_flagged, thread_followed, - thread_unfollowed, thread_voted, + thread_unfollowed ) from openedx.core.djangoapps.user_api.accounts.api import get_account_settings -from openedx.core.lib.exceptions import ( - CourseNotFoundError, - DiscussionNotFoundError, - PageNotFoundError, -) +from openedx.core.lib.exceptions import CourseNotFoundError, DiscussionNotFoundError, PageNotFoundError from xmodule.course_block import CourseBlock from xmodule.modulestore import ModuleStoreEnum from xmodule.modulestore.django import modulestore @@ -94,27 +88,21 @@ from ..django_comment_client.base.views import ( track_comment_created_event, track_comment_deleted_event, - track_discussion_reported_event, - track_discussion_unreported_event, - track_forum_search_event, track_thread_created_event, track_thread_deleted_event, - track_thread_followed_event, track_thread_viewed_event, track_voted_event, + track_discussion_reported_event, + track_discussion_unreported_event, + track_forum_search_event, track_thread_followed_event ) from ..django_comment_client.utils import ( get_group_id_for_user, get_user_role_names, has_discussion_privileges, - is_commentable_divided, -) -from .exceptions import ( - CommentNotFoundError, - DiscussionBlackOutException, - DiscussionDisabledError, - ThreadNotFoundError, + is_commentable_divided ) +from .exceptions import CommentNotFoundError, DiscussionBlackOutException, DiscussionDisabledError, ThreadNotFoundError from .forms import CommentActionsForm, ThreadActionsForm, UserOrdering from .pagination import DiscussionAPIPagination from .permissions import ( @@ -122,7 +110,7 @@ can_take_action_on_spam, get_editable_fields, get_initializable_comment_fields, - get_initializable_thread_fields, + get_initializable_thread_fields ) from .serializers import ( CommentSerializer, @@ -131,23 +119,20 @@ ThreadSerializer, TopicOrdering, UserStatsSerializer, - get_context, + get_context ) from .utils import ( AttributeDict, add_stats_for_users_with_no_discussion_content, - can_user_notify_all_learners, create_blocks_params, discussion_open_for_user, - get_captcha_site_key_by_platform, get_usernames_for_course, get_usernames_from_search_string, - is_captcha_enabled, - is_posting_allowed, set_attribute, + is_posting_allowed, + can_user_notify_all_learners, is_captcha_enabled, get_captcha_site_key_by_platform ) -log = logging.getLogger(__name__) User = get_user_model() ThreadType = Literal["discussion", "question"] @@ -181,14 +166,11 @@ class DiscussionEntity(Enum): """ Enum for different types of discussion related entities """ - - thread = "thread" - comment = "comment" + thread = 'thread' + comment = 'comment' -def _get_course( - course_key: CourseKey, user: User, check_tab: bool = True -) -> CourseBlock: +def _get_course(course_key: CourseKey, user: User, check_tab: bool = True) -> CourseBlock: """ Get the course block, raising CourseNotFoundError if the course is not found or the user cannot access forums for the course, and DiscussionDisabledError if the @@ -206,16 +188,14 @@ def _get_course( CourseBlock: course object """ try: - course = get_course_with_access( - user, "load", course_key, check_if_enrolled=True - ) + course = get_course_with_access(user, 'load', course_key, check_if_enrolled=True) except (Http404, CourseAccessRedirect) as err: # Convert 404s into CourseNotFoundErrors. # Raise course not found if the user cannot access the course raise CourseNotFoundError("Course not found.") from err if check_tab: - discussion_tab = CourseTabList.get_tab_by_type(course.tabs, "discussion") + discussion_tab = CourseTabList.get_tab_by_type(course.tabs, 'discussion') if not (discussion_tab and discussion_tab.is_enabled(course, user)): raise DiscussionDisabledError("Discussion is disabled for the course.") @@ -236,34 +216,22 @@ def _get_thread_and_context(request, thread_id, retrieve_kwargs=None, course_id= retrieve_kwargs["with_responses"] = False if "mark_as_read" not in retrieve_kwargs: retrieve_kwargs["mark_as_read"] = False - cc_thread = Thread(id=thread_id).retrieve( - course_id=course_id, **retrieve_kwargs - ) + cc_thread = Thread(id=thread_id).retrieve(course_id=course_id, **retrieve_kwargs) course_key = CourseKey.from_string(cc_thread["course_id"]) course = _get_course(course_key, request.user) context = get_context(course, request, cc_thread) - if ( - retrieve_kwargs.get("flagged_comments") - and not context["has_moderation_privilege"] - ): + if retrieve_kwargs.get("flagged_comments") and not context["has_moderation_privilege"]: raise ValidationError("Only privileged users can request flagged comments") course_discussion_settings = CourseDiscussionSettings.get(course_key) if ( - not context["has_moderation_privilege"] - and cc_thread["group_id"] - and is_commentable_divided( - course.id, cc_thread["commentable_id"], course_discussion_settings - ) + not context["has_moderation_privilege"] and + cc_thread["group_id"] and + is_commentable_divided(course.id, cc_thread["commentable_id"], course_discussion_settings) ): - requester_group_id = get_group_id_for_user( - request.user, course_discussion_settings - ) - if ( - requester_group_id is not None - and cc_thread["group_id"] != requester_group_id - ): + requester_group_id = get_group_id_for_user(request.user, course_discussion_settings) + if requester_group_id is not None and cc_thread["group_id"] != requester_group_id: raise ThreadNotFoundError("Thread not found.") return cc_thread, context except CommentClientRequestError as err: @@ -296,8 +264,8 @@ def _is_user_author_or_privileged(cc_content, context): Boolean """ return ( - context["has_moderation_privilege"] - or context["cc_requester"]["id"] == cc_content["user_id"] + context["has_moderation_privilege"] or + context["cc_requester"]["id"] == cc_content["user_id"] ) @@ -307,13 +275,11 @@ def get_thread_list_url(request, course_key, topic_id_list=None, following=False """ path = reverse("thread-list") query_list = ( - [("course_id", str(course_key))] - + [("topic_id", topic_id) for topic_id in topic_id_list or []] - + ([("following", following)] if following else []) - ) - return request.build_absolute_uri( - urlunparse(("", "", path, "", urlencode(query_list), "")) + [("course_id", str(course_key))] + + [("topic_id", topic_id) for topic_id in topic_id_list or []] + + ([("following", following)] if following else []) ) + return request.build_absolute_uri(urlunparse(("", "", path, "", urlencode(query_list), ""))) def get_course(request, course_key, check_tab=True): @@ -358,19 +324,18 @@ def _format_datetime(dt): the substitution... though really, that would probably break mobile client parsing of the dates as well. :-P """ - return dt.isoformat().replace("+00:00", "Z") + return dt.isoformat().replace('+00:00', 'Z') course = _get_course(course_key, request.user, check_tab=check_tab) user_roles = get_user_role_names(request.user, course_key) course_config = DiscussionsConfiguration.get(course_key) EDIT_REASON_CODES = getattr(settings, "DISCUSSION_MODERATION_EDIT_REASON_CODES", {}) - CLOSE_REASON_CODES = getattr( - settings, "DISCUSSION_MODERATION_CLOSE_REASON_CODES", {} - ) + CLOSE_REASON_CODES = getattr(settings, "DISCUSSION_MODERATION_CLOSE_REASON_CODES", {}) is_posting_enabled = is_posting_allowed( - course_config.posting_restrictions, course.get_discussion_blackout_datetimes() + course_config.posting_restrictions, + course.get_discussion_blackout_datetimes() ) - discussion_tab = CourseTabList.get_tab_by_type(course.tabs, "discussion") + discussion_tab = CourseTabList.get_tab_by_type(course.tabs, 'discussion') is_course_staff = CourseStaffRole(course_key).has_user(request.user) is_course_admin = CourseInstructorRole(course_key).has_user(request.user) return { @@ -384,9 +349,7 @@ def _format_datetime(dt): for blackout in course.get_discussion_blackout_datetimes() ], "thread_list_url": get_thread_list_url(request, course_key), - "following_thread_list_url": get_thread_list_url( - request, course_key, following=True - ), + "following_thread_list_url": get_thread_list_url(request, course_key, following=True), "topics_url": request.build_absolute_uri( reverse("course_topics", kwargs={"course_id": course_key}) ), @@ -394,23 +357,18 @@ def _format_datetime(dt): "allow_anonymous_to_peers": course.allow_anonymous_to_peers, "user_roles": user_roles, "has_bulk_delete_privileges": can_take_action_on_spam(request.user, course_key), - "has_moderation_privileges": bool( - user_roles - & { - FORUM_ROLE_ADMINISTRATOR, - FORUM_ROLE_MODERATOR, - FORUM_ROLE_COMMUNITY_TA, - } - ), + "has_moderation_privileges": bool(user_roles & { + FORUM_ROLE_ADMINISTRATOR, + FORUM_ROLE_MODERATOR, + FORUM_ROLE_COMMUNITY_TA, + }), "is_group_ta": bool(user_roles & {FORUM_ROLE_GROUP_MODERATOR}), "is_user_admin": request.user.is_staff, "is_course_staff": is_course_staff, "is_course_admin": is_course_admin, "provider": course_config.provider_type, "enable_in_context": course_config.enable_in_context, - "group_at_subsection": course_config.plugin_configuration.get( - "group_at_subsection", False - ), + "group_at_subsection": course_config.plugin_configuration.get("group_at_subsection", False), "edit_reasons": [ {"code": reason_code, "label": label} for (reason_code, label) in EDIT_REASON_CODES.items() @@ -419,23 +377,17 @@ def _format_datetime(dt): {"code": reason_code, "label": label} for (reason_code, label) in CLOSE_REASON_CODES.items() ], - "show_discussions": bool( - discussion_tab and discussion_tab.is_enabled(course, request.user) - ), - "is_notify_all_learners_enabled": can_user_notify_all_learners( + 'show_discussions': bool(discussion_tab and discussion_tab.is_enabled(course, request.user)), + 'is_notify_all_learners_enabled': can_user_notify_all_learners( user_roles, is_course_staff, is_course_admin ), - "captcha_settings": { - "enabled": is_captcha_enabled(course_key), - "site_key": get_captcha_site_key_by_platform("web"), + 'captcha_settings': { + 'enabled': is_captcha_enabled(course_key), + 'site_key': get_captcha_site_key_by_platform('web'), }, "is_email_verified": request.user.is_active, - "only_verified_users_can_post": ONLY_VERIFIED_USERS_CAN_POST.is_enabled( - course_key - ), - "content_creation_rate_limited": is_content_creation_rate_limited( - request, course_key, increment=False - ), + "only_verified_users_can_post": ONLY_VERIFIED_USERS_CAN_POST.is_enabled(course_key), + "content_creation_rate_limited": is_content_creation_rate_limited(request, course_key, increment=False), } @@ -488,7 +440,7 @@ def convert(text): return text def alphanum_key(key): - return [convert(c) for c in re.split("([0-9]+)", key)] + return [convert(c) for c in re.split('([0-9]+)', key)] return sorted(category_list, key=alphanum_key) @@ -530,7 +482,7 @@ def get_non_courseware_topics( course_key: CourseKey, course: CourseBlock, topic_ids: Optional[List[str]], - thread_counts: Dict[str, Dict[str, int]], + thread_counts: Dict[str, Dict[str, int]] ) -> Tuple[List[Dict], Set[str]]: """ Returns a list of topic trees that are not linked to courseware. @@ -554,17 +506,13 @@ def get_non_courseware_topics( existing_topic_ids = set() topics = list(course.discussion_topics.items()) for name, entry in topics: - if not topic_ids or entry["id"] in topic_ids: + if not topic_ids or entry['id'] in topic_ids: discussion_topic = DiscussionTopic( - entry["id"], - name, - get_thread_list_url(request, course_key, [entry["id"]]), + entry["id"], name, get_thread_list_url(request, course_key, [entry["id"]]), None, - thread_counts.get(entry["id"]), - ) - non_courseware_topics.append( - DiscussionTopicSerializer(discussion_topic).data + thread_counts.get(entry["id"]) ) + non_courseware_topics.append(DiscussionTopicSerializer(discussion_topic).data) if topic_ids and entry["id"] in topic_ids: existing_topic_ids.add(entry["id"]) @@ -572,9 +520,7 @@ def get_non_courseware_topics( return non_courseware_topics, existing_topic_ids -def get_course_topics( - request: Request, course_key: CourseKey, topic_ids: Optional[Set[str]] = None -): +def get_course_topics(request: Request, course_key: CourseKey, topic_ids: Optional[Set[str]] = None): """ Returns the course topic listing for the given course and user; filtered by 'topic_ids' list if given. @@ -598,25 +544,15 @@ def get_course_topics( courseware_topics, existing_courseware_topic_ids = get_courseware_topics( request, course_key, course, topic_ids, thread_counts ) - non_courseware_topics, existing_non_courseware_topic_ids = ( - get_non_courseware_topics( - request, - course_key, - course, - topic_ids, - thread_counts, - ) + non_courseware_topics, existing_non_courseware_topic_ids = get_non_courseware_topics( + request, course_key, course, topic_ids, thread_counts, ) if topic_ids: - not_found_topic_ids = topic_ids - ( - existing_courseware_topic_ids | existing_non_courseware_topic_ids - ) + not_found_topic_ids = topic_ids - (existing_courseware_topic_ids | existing_non_courseware_topic_ids) if not_found_topic_ids: raise DiscussionNotFoundError( - "Discussion not found for '{}'.".format( - ", ".join(str(id) for id in not_found_topic_ids) - ) + "Discussion not found for '{}'.".format(", ".join(str(id) for id in not_found_topic_ids)) ) return { @@ -631,19 +567,17 @@ def get_v2_non_courseware_topics_as_v1(request, course_key, topics): """ non_courseware_topics = [] for topic in topics: - if topic.get("usage_key", "") is None: - for key in ["usage_key", "enabled_in_context"]: + if topic.get('usage_key', '') is None: + for key in ['usage_key', 'enabled_in_context']: topic.pop(key) - topic.update( - { - "children": [], - "thread_list_url": get_thread_list_url( - request, - course_key, - topic.get("id"), - ), - } - ) + topic.update({ + 'children': [], + 'thread_list_url': get_thread_list_url( + request, + course_key, + topic.get('id'), + ) + }) non_courseware_topics.append(topic) return non_courseware_topics @@ -655,25 +589,23 @@ def get_v2_courseware_topics_as_v1(request, course_key, sequentials, topics): courseware_topics = [] for sequential in sequentials: children = [] - for child in sequential.get("children", []): + for child in sequential.get('children', []): for topic in topics: - if child == topic.get("usage_key"): - topic.update( - { - "children": [], - "thread_list_url": get_thread_list_url( - request, - course_key, - [topic.get("id")], - ), - } - ) - topic.pop("enabled_in_context") + if child == topic.get('usage_key'): + topic.update({ + 'children': [], + 'thread_list_url': get_thread_list_url( + request, + course_key, + [topic.get('id')], + ) + }) + topic.pop('enabled_in_context') children.append(AttributeDict(topic)) discussion_topic = DiscussionTopic( None, - sequential.get("display_name"), + sequential.get('display_name'), get_thread_list_url( request, course_key, @@ -686,7 +618,7 @@ def get_v2_courseware_topics_as_v1(request, course_key, sequentials, topics): courseware_topics = [ courseware_topic for courseware_topic in courseware_topics - if courseware_topic.get("children", []) + if courseware_topic.get('children', []) ] return courseware_topics @@ -703,21 +635,20 @@ def get_v2_course_topics_as_v1( blocks_params = create_blocks_params(course_usage_key, request.user) blocks = get_blocks( request, - blocks_params["usage_key"], - blocks_params["user"], - blocks_params["depth"], - blocks_params["nav_depth"], - blocks_params["requested_fields"], - blocks_params["block_counts"], - blocks_params["student_view_data"], - blocks_params["return_type"], - blocks_params["block_types_filter"], + blocks_params['usage_key'], + blocks_params['user'], + blocks_params['depth'], + blocks_params['nav_depth'], + blocks_params['requested_fields'], + blocks_params['block_counts'], + blocks_params['student_view_data'], + blocks_params['return_type'], + blocks_params['block_types_filter'], hide_access_denials=False, - )["blocks"] + )['blocks'] - sequentials = [ - value for _, value in blocks.items() if value.get("type") == "sequential" - ] + sequentials = [value for _, value in blocks.items() + if value.get('type') == "sequential"] topics = get_course_topics_v2(course_key, request.user, topic_ids) non_courseware_topics = get_v2_non_courseware_topics_as_v1( @@ -774,29 +705,24 @@ def get_course_topics_v2( # Check access to the course store = modulestore() _get_course(course_key, user=user, check_tab=False) - user_is_privileged = ( - user.is_staff - or user.roles.filter( - course_id=course_key, - name__in=[ - FORUM_ROLE_MODERATOR, - FORUM_ROLE_COMMUNITY_TA, - FORUM_ROLE_ADMINISTRATOR, - ], - ).exists() - ) + user_is_privileged = user.is_staff or user.roles.filter( + course_id=course_key, + name__in=[ + FORUM_ROLE_MODERATOR, + FORUM_ROLE_COMMUNITY_TA, + FORUM_ROLE_ADMINISTRATOR, + ] + ).exists() with store.branch_setting(ModuleStoreEnum.Branch.draft_preferred, course_key): blocks = store.get_items( course_key, - qualifiers={"category": "vertical"}, - fields=["usage_key", "discussion_enabled", "display_name"], + qualifiers={'category': 'vertical'}, + fields=['usage_key', 'discussion_enabled', 'display_name'], ) accessible_vertical_keys = [] for block in blocks: - if block.discussion_enabled and ( - not block.visible_to_staff_only or user_is_privileged - ): + if block.discussion_enabled and (not block.visible_to_staff_only or user_is_privileged): accessible_vertical_keys.append(block.usage_key) accessible_vertical_keys.append(None) @@ -806,13 +732,9 @@ def get_course_topics_v2( ) if user_is_privileged: - topics_query = topics_query.filter( - Q(usage_key__in=accessible_vertical_keys) | Q(enabled_in_context=False) - ) + topics_query = topics_query.filter(Q(usage_key__in=accessible_vertical_keys) | Q(enabled_in_context=False)) else: - topics_query = topics_query.filter( - usage_key__in=accessible_vertical_keys, enabled_in_context=True - ) + topics_query = topics_query.filter(usage_key__in=accessible_vertical_keys, enabled_in_context=True) if topic_ids: topics_query = topics_query.filter(external_id__in=topic_ids) @@ -824,13 +746,11 @@ def get_course_topics_v2( reverse=True, ) elif order_by == TopicOrdering.NAME: - topics_query = topics_query.order_by("title") + topics_query = topics_query.order_by('title') else: - topics_query = topics_query.order_by("ordering") + topics_query = topics_query.order_by('ordering') - topics_data = DiscussionTopicSerializerV2( - topics_query, many=True, context={"thread_counts": thread_counts} - ).data + topics_data = DiscussionTopicSerializerV2(topics_query, many=True, context={"thread_counts": thread_counts}).data return [ topic_data for topic_data in topics_data @@ -857,7 +777,7 @@ def _get_user_profile_dict(request, usernames): else: username_list = [] user_profile_details = get_account_settings(request, username_list) - return {user["username"]: user for user in user_profile_details} + return {user['username']: user for user in user_profile_details} def _user_profile(user_profile): @@ -865,7 +785,11 @@ def _user_profile(user_profile): Returns the user profile object. For now, this just comprises the profile_image details. """ - return {"profile": {"image": user_profile["profile_image"]}} + return { + 'profile': { + 'image': user_profile['profile_image'] + } + } def _get_users(discussion_entity_type, discussion_entity, username_profile_dict): @@ -883,28 +807,22 @@ def _get_users(discussion_entity_type, discussion_entity, username_profile_dict) A dict of users with username as key and user profile details as value. """ users = {} - if discussion_entity["author"]: - user_profile = username_profile_dict.get(discussion_entity["author"]) + if discussion_entity['author']: + user_profile = username_profile_dict.get(discussion_entity['author']) if user_profile: - users[discussion_entity["author"]] = _user_profile(user_profile) + users[discussion_entity['author']] = _user_profile(user_profile) if ( discussion_entity_type == DiscussionEntity.comment - and discussion_entity["endorsed"] - and discussion_entity["endorsed_by"] + and discussion_entity['endorsed'] + and discussion_entity['endorsed_by'] ): - users[discussion_entity["endorsed_by"]] = _user_profile( - username_profile_dict[discussion_entity["endorsed_by"]] - ) + users[discussion_entity['endorsed_by']] = _user_profile(username_profile_dict[discussion_entity['endorsed_by']]) return users def _add_additional_response_fields( - request, - serialized_discussion_entities, - usernames, - discussion_entity_type, - include_profile_image, + request, serialized_discussion_entities, usernames, discussion_entity_type, include_profile_image ): """ Adds additional data to serialized discussion thread/comment. @@ -922,13 +840,9 @@ def _add_additional_response_fields( A list of serialized discussion thread/comment with additional data if requested. """ if include_profile_image: - username_profile_dict = _get_user_profile_dict( - request, usernames=",".join(usernames) - ) + username_profile_dict = _get_user_profile_dict(request, usernames=','.join(usernames)) for discussion_entity in serialized_discussion_entities: - discussion_entity["users"] = _get_users( - discussion_entity_type, discussion_entity, username_profile_dict - ) + discussion_entity['users'] = _get_users(discussion_entity_type, discussion_entity, username_profile_dict) return serialized_discussion_entities @@ -937,12 +851,10 @@ def _include_profile_image(requested_fields): """ Returns True if requested_fields list has 'profile_image' entity else False """ - return requested_fields and "profile_image" in requested_fields + return requested_fields and 'profile_image' in requested_fields -def _serialize_discussion_entities( - request, context, discussion_entities, requested_fields, discussion_entity_type -): +def _serialize_discussion_entities(request, context, discussion_entities, requested_fields, discussion_entity_type): """ It serializes Discussion Entity (Thread or Comment) and add additional data if requested. @@ -973,19 +885,14 @@ def _serialize_discussion_entities( results.append(serialized_entity) if include_profile_image: + if serialized_entity['author'] and serialized_entity['author'] not in usernames: + usernames.append(serialized_entity['author']) if ( - serialized_entity["author"] - and serialized_entity["author"] not in usernames + 'endorsed' in serialized_entity and serialized_entity['endorsed'] and + 'endorsed_by' in serialized_entity and + serialized_entity['endorsed_by'] and serialized_entity['endorsed_by'] not in usernames ): - usernames.append(serialized_entity["author"]) - if ( - "endorsed" in serialized_entity - and serialized_entity["endorsed"] - and "endorsed_by" in serialized_entity - and serialized_entity["endorsed_by"] - and serialized_entity["endorsed_by"] not in usernames - ): - usernames.append(serialized_entity["endorsed_by"]) + usernames.append(serialized_entity['endorsed_by']) results = _add_additional_response_fields( request, results, usernames, discussion_entity_type, include_profile_image @@ -1009,7 +916,6 @@ def get_thread_list( order_direction: Literal["desc"] = "desc", requested_fields: Optional[List[Literal["profile_image"]]] = None, count_flagged: bool = None, - show_deleted: bool = False, ): """ Return the list of all discussion threads pertaining to the given course @@ -1053,31 +959,20 @@ def get_thread_list( CourseNotFoundError: if the requesting user does not have access to the requested course PageNotFoundError: if page requested is beyond the last """ - exclusive_param_count = sum( - 1 for param in [topic_id_list, text_search, following] if param - ) + exclusive_param_count = sum(1 for param in [topic_id_list, text_search, following] if param) if exclusive_param_count > 1: # pragma: no cover - raise ValueError( - "More than one mutually exclusive param passed to get_thread_list" - ) + raise ValueError("More than one mutually exclusive param passed to get_thread_list") - cc_map = { - "last_activity_at": "activity", - "comment_count": "comments", - "vote_count": "votes", - } + cc_map = {"last_activity_at": "activity", "comment_count": "comments", "vote_count": "votes"} if order_by not in cc_map: - raise ValidationError( - { - "order_by": [ - f"Invalid value. '{order_by}' must be 'last_activity_at', 'comment_count', or 'vote_count'" - ] - } - ) + raise ValidationError({ + "order_by": + [f"Invalid value. '{order_by}' must be 'last_activity_at', 'comment_count', or 'vote_count'"] + }) if order_direction != "desc": - raise ValidationError( - {"order_direction": [f"Invalid value. '{order_direction}' must be 'desc'"]} - ) + raise ValidationError({ + "order_direction": [f"Invalid value. '{order_direction}' must be 'desc'"] + }) course = _get_course(course_key, request.user) context = get_context(course, request) @@ -1089,21 +984,13 @@ def get_thread_list( except User.DoesNotExist: # Raising an error for a missing user leaks the presence of a username, # so just return an empty response. - return DiscussionAPIPagination(request, 0, 1).get_paginated_response( - { - "results": [], - "text_search_rewrite": None, - } - ) + return DiscussionAPIPagination(request, 0, 1).get_paginated_response({ + "results": [], + "text_search_rewrite": None, + }) if count_flagged and not context["has_moderation_privilege"]: - raise PermissionDenied( - "`count_flagged` can only be set by users with moderator access or higher." - ) - if show_deleted and not context["has_moderation_privilege"]: - raise PermissionDenied( - "`show_deleted` can only be set by users with moderator access or higher." - ) + raise PermissionDenied("`count_flagged` can only be set by users with moderator access or higher.") group_id = None allowed_roles = [ @@ -1123,9 +1010,7 @@ def get_thread_list( not context["has_moderation_privilege"] or request.user.id in context["ta_user_ids"] ): - group_id = get_group_id_for_user( - request.user, CourseDiscussionSettings.get(course.id) - ) + group_id = get_group_id_for_user(request.user, CourseDiscussionSettings.get(course.id)) query_params = { "user_id": str(request.user.id), @@ -1138,24 +1023,21 @@ def get_thread_list( "flagged": flagged, "thread_type": thread_type, "count_flagged": count_flagged, - "show_deleted": show_deleted, } if view: if view in ["unread", "unanswered", "unresponded"]: query_params[view] = "true" else: - raise ValidationError( - {"view": [f"Invalid value. '{view}' must be 'unread' or 'unanswered'"]} - ) + raise ValidationError({ + "view": [f"Invalid value. '{view}' must be 'unread' or 'unanswered'"] + }) if following: paginated_results = context["cc_requester"].subscribed_threads(query_params) else: query_params["course_id"] = str(course.id) - query_params["commentable_ids"] = ( - ",".join(topic_id_list) if topic_id_list else None - ) + query_params["commentable_ids"] = ",".join(topic_id_list) if topic_id_list else None query_params["text"] = text_search paginated_results = Thread.search(query_params) # The comments service returns the last page of results if the requested @@ -1165,25 +1047,19 @@ def get_thread_list( raise PageNotFoundError("Page not found (No results on this page).") results = _serialize_discussion_entities( - request, - context, - paginated_results.collection, - requested_fields, - DiscussionEntity.thread, + request, context, paginated_results.collection, requested_fields, DiscussionEntity.thread ) paginator = DiscussionAPIPagination( request, paginated_results.page, paginated_results.num_pages, - paginated_results.thread_count, - ) - return paginator.get_paginated_response( - { - "results": results, - "text_search_rewrite": paginated_results.corrected_text, - } + paginated_results.thread_count ) + return paginator.get_paginated_response({ + "results": results, + "text_search_rewrite": paginated_results.corrected_text, + }) def get_learner_active_thread_list(request, course_key, query_params): @@ -1278,101 +1154,49 @@ def get_learner_active_thread_list(request, course_key, query_params): course = _get_course(course_key, request.user) context = get_context(course, request) - group_id = query_params.get("group_id", None) - user_id = query_params.get("user_id", None) - count_flagged = query_params.get("count_flagged", None) - show_deleted = query_params.get("show_deleted", False) - if isinstance(show_deleted, str): - show_deleted = show_deleted.lower() == "true" - + group_id = query_params.get('group_id', None) + user_id = query_params.get('user_id', None) + count_flagged = query_params.get('count_flagged', None) if user_id is None: - return Response( - {"detail": "Invalid user id"}, status=status.HTTP_400_BAD_REQUEST - ) + return Response({'detail': 'Invalid user id'}, status=status.HTTP_400_BAD_REQUEST) if count_flagged and not context["has_moderation_privilege"]: - raise PermissionDenied( - "count_flagged can only be set by users with moderation roles." - ) + raise PermissionDenied("count_flagged can only be set by users with moderation roles.") if "flagged" in query_params.keys() and not context["has_moderation_privilege"]: raise PermissionDenied("Flagged filter is only available for moderators") - if show_deleted and not context["has_moderation_privilege"]: - raise PermissionDenied( - "show_deleted can only be set by users with moderation roles." - ) if group_id is None: comment_client_user = comment_client.User(id=user_id, course_id=course_key) else: - comment_client_user = comment_client.User( - id=user_id, course_id=course_key, group_id=group_id - ) + comment_client_user = comment_client.User(id=user_id, course_id=course_key, group_id=group_id) try: threads, page, num_pages = comment_client_user.active_threads(query_params) threads = set_attribute(threads, "pinned", False) - - # This portion below is temporary until we migrate to forum v2 - filtered_threads = [] - for thread in threads: - try: - forum_thread = forum_api.get_thread( - thread.get("id"), course_id=str(course_key) - ) - is_deleted = forum_thread.get("is_deleted", False) - - if show_deleted and is_deleted: - thread["is_deleted"] = True - thread["deleted_at"] = forum_thread.get("deleted_at") - thread["deleted_by"] = forum_thread.get("deleted_by") - filtered_threads.append(thread) - elif not show_deleted and not is_deleted: - filtered_threads.append(thread) - except Exception as e: # pylint: disable=broad-exception-caught - log.warning( - "Failed to check thread %s deletion status: %s", thread.get("id"), e - ) - if not show_deleted: # Fail safe: include thread for regular users - filtered_threads.append(thread) - results = _serialize_discussion_entities( - request, - context, - filtered_threads, - {"profile_image"}, - DiscussionEntity.thread, + request, context, threads, {'profile_image'}, DiscussionEntity.thread ) paginator = DiscussionAPIPagination( - request, page, num_pages, len(filtered_threads) - ) - return paginator.get_paginated_response( - { - "results": results, - } + request, + page, + num_pages, + len(threads) ) + return paginator.get_paginated_response({ + "results": results, + }) except CommentClient500Error: return DiscussionAPIPagination( request, page_num=1, num_pages=0, - ).get_paginated_response( - { - "results": [], - } - ) + ).get_paginated_response({ + "results": [], + }) -def get_comment_list( - request, - thread_id, - endorsed, - page, - page_size, - flagged=False, - requested_fields=None, - merge_question_type_responses=False, - show_deleted=False, -): +def get_comment_list(request, thread_id, endorsed, page, page_size, flagged=False, requested_fields=None, + merge_question_type_responses=False): """ Return the list of comments in the given thread. @@ -1402,7 +1226,7 @@ def get_comment_list( discussion.rest_api.views.CommentViewSet for more detail. """ response_skip = page_size * (page - 1) - reverse_order = request.GET.get("reverse_order", False) + reverse_order = request.GET.get('reverse_order', False) from_mfe_sidebar = request.GET.get("enable_in_context_sidebar", False) cc_thread, context = _get_thread_and_context( request, @@ -1415,23 +1239,19 @@ def get_comment_list( "response_skip": response_skip, "response_limit": page_size, "reverse_order": reverse_order, - "merge_question_type_responses": merge_question_type_responses, - }, + "merge_question_type_responses": merge_question_type_responses + } ) # Responses to discussion threads cannot be separated by endorsed, but # responses to question threads must be separated by endorsed due to the # existing comments service interface if cc_thread["thread_type"] == "question" and not merge_question_type_responses: if endorsed is None: # lint-amnesty, pylint: disable=no-else-raise - raise ValidationError( - {"endorsed": ["This field is required for question threads."]} - ) + raise ValidationError({"endorsed": ["This field is required for question threads."]}) elif endorsed: # CS does not apply resp_skip and resp_limit to endorsed responses # of a question post - responses = cc_thread["endorsed_responses"][ - response_skip: (response_skip + page_size) - ] + responses = cc_thread["endorsed_responses"][response_skip:(response_skip + page_size)] resp_total = len(cc_thread["endorsed_responses"]) else: responses = cc_thread["non_endorsed_responses"] @@ -1440,11 +1260,7 @@ def get_comment_list( if not merge_question_type_responses: if endorsed is not None: raise ValidationError( - { - "endorsed": [ - "This field may not be specified for discussion threads." - ] - } + {"endorsed": ["This field may not be specified for discussion threads."]} ) responses = cc_thread["children"] resp_total = cc_thread["resp_total"] @@ -1456,21 +1272,9 @@ def get_comment_list( raise PageNotFoundError("Page not found (No results on this page).") num_pages = (resp_total + page_size - 1) // page_size if resp_total else 1 - if not show_deleted: - responses = [ - response for response in responses if not response.get("is_deleted", False) - ] - else: - if not context["has_moderation_privilege"]: - raise PermissionDenied( - "`show_deleted` can only be set by users with moderation roles." - ) - - results = _serialize_discussion_entities( - request, context, responses, requested_fields, DiscussionEntity.comment - ) + results = _serialize_discussion_entities(request, context, responses, requested_fields, DiscussionEntity.comment) - paginator = DiscussionAPIPagination(request, page, num_pages, len(responses)) + paginator = DiscussionAPIPagination(request, page, num_pages, resp_total) track_thread_viewed_event(request, context["course"], cc_thread, from_mfe_sidebar) return paginator.get_paginated_response(results) @@ -1488,9 +1292,7 @@ def _check_fields(allowed_fields, data, message): ValidationError if the given data contains a key that is not in allowed_fields """ - non_allowed_fields = { - field: [message] for field in data.keys() if field not in allowed_fields - } + non_allowed_fields = {field: [message] for field in data.keys() if field not in allowed_fields} if non_allowed_fields: raise ValidationError(non_allowed_fields) @@ -1512,7 +1314,7 @@ def _check_initializable_thread_fields(data, context): _check_fields( get_initializable_thread_fields(context), data, - "This field is not initializable.", + "This field is not initializable." ) @@ -1533,7 +1335,7 @@ def _check_initializable_comment_fields(data, context): _check_fields( get_initializable_comment_fields(context), data, - "This field is not initializable.", + "This field is not initializable." ) @@ -1543,40 +1345,28 @@ def _check_editable_fields(cc_content, data, context): editable by the requesting user """ _check_fields( - get_editable_fields(cc_content, context), data, "This field is not editable." + get_editable_fields(cc_content, context), + data, + "This field is not editable." ) -def _do_extra_actions( - api_content, cc_content, request_fields, actions_form, context, request -): +def _do_extra_actions(api_content, cc_content, request_fields, actions_form, context, request): """ Perform any necessary additional actions related to content creation or update that require a separate comments service request. """ for field, form_value in actions_form.cleaned_data.items(): - if ( - field in request_fields - and field in api_content - and form_value != api_content[field] - ): + if field in request_fields and field in api_content and form_value != api_content[field]: api_content[field] = form_value if field == "following": - _handle_following_field( - form_value, context["cc_requester"], cc_content, request - ) + _handle_following_field(form_value, context["cc_requester"], cc_content, request) elif field == "abuse_flagged": - _handle_abuse_flagged_field( - form_value, context["cc_requester"], cc_content, request - ) + _handle_abuse_flagged_field(form_value, context["cc_requester"], cc_content, request) elif field == "voted": - _handle_voted_field( - form_value, cc_content, api_content, request, context - ) + _handle_voted_field(form_value, cc_content, api_content, request, context) elif field == "read": - _handle_read_field( - api_content, form_value, context["cc_requester"], cc_content - ) + _handle_read_field(api_content, form_value, context["cc_requester"], cc_content) elif field == "pinned": _handle_pinned_field(form_value, cc_content, context["cc_requester"]) else: @@ -1586,7 +1376,7 @@ def _do_extra_actions( def _handle_following_field(form_value, user, cc_content, request): """follow/unfollow thread for the user""" course_key = CourseKey.from_string(cc_content.course_id) - course = get_course_with_access(request.user, "load", course_key) + course = get_course_with_access(request.user, 'load', course_key) if form_value: user.follow(cc_content) else: @@ -1599,19 +1389,15 @@ def _handle_following_field(form_value, user, cc_content, request): def _handle_abuse_flagged_field(form_value, user, cc_content, request): """mark or unmark thread/comment as abused""" course_key = CourseKey.from_string(cc_content.course_id) - course = get_course_with_access(request.user, "load", course_key) + course = get_course_with_access(request.user, 'load', course_key) if form_value: cc_content.flagAbuse(user, cc_content) track_discussion_reported_event(request, course, cc_content) if ENABLE_DISCUSSIONS_MFE.is_enabled(course_key): - if cc_content.type == "thread": - thread_flagged.send( - sender="flag_abuse_for_thread", user=user, post=cc_content - ) + if cc_content.type == 'thread': + thread_flagged.send(sender='flag_abuse_for_thread', user=user, post=cc_content) else: - comment_flagged.send( - sender="flag_abuse_for_comment", user=user, post=cc_content - ) + comment_flagged.send(sender='flag_abuse_for_comment', user=user, post=cc_content) else: remove_all = bool(is_privileged_user(course_key, User.objects.get(id=user.id))) cc_content.unFlagAbuse(user, cc_content, remove_all) @@ -1620,7 +1406,7 @@ def _handle_abuse_flagged_field(form_value, user, cc_content, request): def _handle_voted_field(form_value, cc_content, api_content, request, context): """vote or undo vote on thread/comment""" - signal = thread_voted if cc_content.type == "thread" else comment_voted + signal = thread_voted if cc_content.type == 'thread' else comment_voted signal.send(sender=None, user=context["request"].user, post=cc_content) if form_value: context["cc_requester"].vote(cc_content, "up") @@ -1629,11 +1415,7 @@ def _handle_voted_field(form_value, cc_content, api_content, request, context): context["cc_requester"].unvote(cc_content) api_content["vote_count"] -= 1 track_voted_event( - request, - context["course"], - cc_content, - vote_value="up", - undo_vote=not form_value, + request, context["course"], cc_content, vote_value="up", undo_vote=not form_value ) @@ -1641,7 +1423,7 @@ def _handle_read_field(api_content, form_value, user, cc_content): """ Marks thread as read for the user """ - if form_value and not cc_content["read"]: + if form_value and not cc_content['read']: user.read(cc_content) # When a thread is marked as read, all of its responses and comments # are also marked as read. @@ -1708,35 +1490,24 @@ def create_thread(request, thread_data): context = get_context(course, request) _check_initializable_thread_fields(thread_data, context) discussion_settings = CourseDiscussionSettings.get(course_key) - if "group_id" not in thread_data and is_commentable_divided( - course_key, thread_data.get("topic_id"), discussion_settings + if ( + "group_id" not in thread_data and + is_commentable_divided(course_key, thread_data.get("topic_id"), discussion_settings) ): thread_data = thread_data.copy() thread_data["group_id"] = get_group_id_for_user(user, discussion_settings) serializer = ThreadSerializer(data=thread_data, context=context) actions_form = ThreadActionsForm(thread_data) if not (serializer.is_valid() and actions_form.is_valid()): - raise ValidationError( - dict(list(serializer.errors.items()) + list(actions_form.errors.items())) - ) + raise ValidationError(dict(list(serializer.errors.items()) + list(actions_form.errors.items()))) serializer.save() cc_thread = serializer.instance - thread_created.send( - sender=None, user=user, post=cc_thread, notify_all_learners=notify_all_learners - ) + thread_created.send(sender=None, user=user, post=cc_thread, notify_all_learners=notify_all_learners) api_thread = serializer.data - _do_extra_actions( - api_thread, cc_thread, list(thread_data.keys()), actions_form, context, request - ) + _do_extra_actions(api_thread, cc_thread, list(thread_data.keys()), actions_form, context, request) - track_thread_created_event( - request, - course, - cc_thread, - actions_form.cleaned_data["following"], - from_mfe_sidebar, - notify_all_learners, - ) + track_thread_created_event(request, course, cc_thread, actions_form.cleaned_data["following"], + from_mfe_sidebar, notify_all_learners) return api_thread @@ -1775,30 +1546,15 @@ def create_comment(request, comment_data): serializer = CommentSerializer(data=comment_data, context=context) actions_form = CommentActionsForm(comment_data) if not (serializer.is_valid() and actions_form.is_valid()): - raise ValidationError( - dict(list(serializer.errors.items()) + list(actions_form.errors.items())) - ) + raise ValidationError(dict(list(serializer.errors.items()) + list(actions_form.errors.items()))) context["cc_requester"].follow(cc_thread) serializer.save() cc_comment = serializer.instance comment_created.send(sender=None, user=request.user, post=cc_comment) api_comment = serializer.data - _do_extra_actions( - api_comment, - cc_comment, - list(comment_data.keys()), - actions_form, - context, - request, - ) - track_comment_created_event( - request, - course, - cc_comment, - cc_thread["commentable_id"], - followed=False, - from_mfe_sidebar=from_mfe_sidebar, - ) + _do_extra_actions(api_comment, cc_comment, list(comment_data.keys()), actions_form, context, request) + track_comment_created_event(request, course, cc_comment, cc_thread["commentable_id"], followed=False, + from_mfe_sidebar=from_mfe_sidebar) return api_comment @@ -1820,32 +1576,24 @@ def update_thread(request, thread_id, update_data): The updated thread; see discussion.rest_api.views.ThreadViewSet for more detail. """ - cc_thread, context = _get_thread_and_context( - request, thread_id, retrieve_kwargs={"with_responses": True} - ) + cc_thread, context = _get_thread_and_context(request, thread_id, retrieve_kwargs={"with_responses": True}) _check_editable_fields(cc_thread, update_data, context) - serializer = ThreadSerializer( - cc_thread, data=update_data, partial=True, context=context - ) + serializer = ThreadSerializer(cc_thread, data=update_data, partial=True, context=context) actions_form = ThreadActionsForm(update_data) if not (serializer.is_valid() and actions_form.is_valid()): - raise ValidationError( - dict(list(serializer.errors.items()) + list(actions_form.errors.items())) - ) + raise ValidationError(dict(list(serializer.errors.items()) + list(actions_form.errors.items()))) # Only save thread object if some of the edited fields are in the thread data, not extra actions if set(update_data) - set(actions_form.fields): serializer.save() # signal to update Teams when a user edits a thread thread_edited.send(sender=None, user=request.user, post=cc_thread) api_thread = serializer.data - _do_extra_actions( - api_thread, cc_thread, list(update_data.keys()), actions_form, context, request - ) + _do_extra_actions(api_thread, cc_thread, list(update_data.keys()), actions_form, context, request) # always return read as True (and therefore unread_comment_count=0) as reasonably # accurate shortcut, rather than adding additional processing. - api_thread["read"] = True - api_thread["unread_comment_count"] = 0 + api_thread['read'] = True + api_thread['unread_comment_count'] = 0 return api_thread @@ -1880,27 +1628,16 @@ def update_comment(request, comment_id, update_data): """ cc_comment, context = _get_comment_and_context(request, comment_id) _check_editable_fields(cc_comment, update_data, context) - serializer = CommentSerializer( - cc_comment, data=update_data, partial=True, context=context - ) + serializer = CommentSerializer(cc_comment, data=update_data, partial=True, context=context) actions_form = CommentActionsForm(update_data) if not (serializer.is_valid() and actions_form.is_valid()): - raise ValidationError( - dict(list(serializer.errors.items()) + list(actions_form.errors.items())) - ) + raise ValidationError(dict(list(serializer.errors.items()) + list(actions_form.errors.items()))) # Only save comment object if some of the edited fields are in the comment data, not extra actions if set(update_data) - set(actions_form.fields): serializer.save() comment_edited.send(sender=None, user=request.user, post=cc_comment) api_comment = serializer.data - _do_extra_actions( - api_comment, - cc_comment, - list(update_data.keys()), - actions_form, - context, - request, - ) + _do_extra_actions(api_comment, cc_comment, list(update_data.keys()), actions_form, context, request) _handle_comment_signals(update_data, cc_comment, request.user) return api_comment @@ -1934,9 +1671,7 @@ def get_thread(request, thread_id, requested_fields=None, course_id=None): ) if course_id and course_id != cc_thread.course_id: raise ThreadNotFoundError("Thread not found.") - return _serialize_discussion_entities( - request, context, [cc_thread], requested_fields, DiscussionEntity.thread - )[0] + return _serialize_discussion_entities(request, context, [cc_thread], requested_fields, DiscussionEntity.thread)[0] def get_response_comments(request, comment_id, page, page_size, requested_fields=None): @@ -1964,10 +1699,7 @@ def get_response_comments(request, comment_id, page, page_size, requested_fields """ try: cc_comment = Comment(id=comment_id).retrieve() - reverse_order = request.GET.get("reverse_order", False) - show_deleted = request.GET.get("show_deleted", False) - show_deleted = show_deleted in ["true", "True", True] - + reverse_order = request.GET.get('reverse_order', False) cc_thread, context = _get_thread_and_context( request, cc_comment["thread_id"], @@ -1975,13 +1707,10 @@ def get_response_comments(request, comment_id, page, page_size, requested_fields "with_responses": True, "recursive": True, "reverse_order": reverse_order, - "show_deleted": show_deleted, - }, + } ) if cc_thread["thread_type"] == "question": - thread_responses = itertools.chain( - cc_thread["endorsed_responses"], cc_thread["non_endorsed_responses"] - ) + thread_responses = itertools.chain(cc_thread["endorsed_responses"], cc_thread["non_endorsed_responses"]) else: thread_responses = cc_thread["children"] response_comments = [] @@ -1991,35 +1720,16 @@ def get_response_comments(request, comment_id, page, page_size, requested_fields break response_skip = page_size * (page - 1) - paged_response_comments = response_comments[ - response_skip: (response_skip + page_size) - ] + paged_response_comments = response_comments[response_skip:(response_skip + page_size)] if not paged_response_comments and page != 1: raise PageNotFoundError("Page not found (No results on this page).") - if not show_deleted: - paged_response_comments = [ - response - for response in paged_response_comments - if not response.get("is_deleted", False) - ] - else: - if not context["has_moderation_privilege"]: - raise PermissionDenied( - "`show_deleted` can only be set by users with moderation roles." - ) results = _serialize_discussion_entities( - request, - context, - paged_response_comments, - requested_fields, - DiscussionEntity.comment, + request, context, paged_response_comments, requested_fields, DiscussionEntity.comment ) - comments_count = len(paged_response_comments) - num_pages = ( - (comments_count + page_size - 1) // page_size if comments_count else 1 - ) + comments_count = len(response_comments) + num_pages = (comments_count + page_size - 1) // page_size if comments_count else 1 paginator = DiscussionAPIPagination(request, page, num_pages, comments_count) return paginator.get_paginated_response(results) except CommentClientRequestError as err: @@ -2063,20 +1773,16 @@ def get_user_comments( context = get_context(course, request) if flagged and not context["has_moderation_privilege"]: - raise ValidationError( - "Only privileged users can filter comments by flagged status" - ) + raise ValidationError("Only privileged users can filter comments by flagged status") try: - response = Comment.retrieve_all( - { - "user_id": author.id, - "course_id": str(course_key), - "flagged": flagged, - "page": page, - "per_page": page_size, - } - ) + response = Comment.retrieve_all({ + 'user_id': author.id, + 'course_id': str(course_key), + 'flagged': flagged, + 'page': page, + 'per_page': page_size, + }) except CommentClientRequestError as err: raise CommentNotFoundError("Comment not found") from err @@ -2116,7 +1822,7 @@ def delete_thread(request, thread_id): """ cc_thread, context = _get_thread_and_context(request, thread_id) if can_delete(cc_thread, context): - cc_thread.delete(deleted_by=str(request.user.id)) + cc_thread.delete() thread_deleted.send(sender=None, user=request.user, post=cc_thread) track_thread_deleted_event(request, context["course"], cc_thread) else: @@ -2141,7 +1847,7 @@ def delete_comment(request, comment_id): """ cc_comment, context = _get_comment_and_context(request, comment_id) if can_delete(cc_comment, context): - cc_comment.delete(deleted_by=str(request.user.id)) + cc_comment.delete() comment_deleted.send(sender=None, user=request.user, post=cc_comment) track_comment_deleted_event(request, context["course"], cc_comment) else: @@ -2173,10 +1879,7 @@ def get_course_discussion_user_stats( """ course_key = CourseKey.from_string(course_key_str) - is_privileged = ( - has_discussion_privileges(user=request.user, course_id=course_key) - or request.user.is_staff - ) + is_privileged = has_discussion_privileges(user=request.user, course_id=course_key) or request.user.is_staff if is_privileged: order_by = order_by or UserOrdering.BY_FLAGS else: @@ -2185,35 +1888,30 @@ def get_course_discussion_user_stats( raise ValidationError({"order_by": "Invalid value"}) params = { - "sort_key": str(order_by), - "page": page, - "per_page": page_size, + 'sort_key': str(order_by), + 'page': page, + 'per_page': page_size, } comma_separated_usernames = matched_users_count = matched_users_pages = None if username_search_string: - comma_separated_usernames, matched_users_count, matched_users_pages = ( - get_usernames_from_search_string( - course_key, username_search_string, page, page_size - ) + comma_separated_usernames, matched_users_count, matched_users_pages = get_usernames_from_search_string( + course_key, username_search_string, page, page_size ) search_event_data = { - "query": username_search_string, - "search_type": "Learner", - "page": params.get("page"), - "sort_key": params.get("sort_key"), - "total_results": matched_users_count, + 'query': username_search_string, + 'search_type': 'Learner', + 'page': params.get('page'), + 'sort_key': params.get('sort_key'), + 'total_results': matched_users_count, } course = _get_course(course_key, request.user) track_forum_search_event(request, course, search_event_data) - if not comma_separated_usernames: - return DiscussionAPIPagination(request, 0, 1).get_paginated_response( - { - "results": [], - } - ) + return DiscussionAPIPagination(request, 0, 1).get_paginated_response({ + "results": [], + }) - params["usernames"] = comma_separated_usernames + params['usernames'] = comma_separated_usernames course_stats_response = get_course_user_stats(course_key, params) @@ -2233,429 +1931,71 @@ def get_course_discussion_user_stats( paginator = DiscussionAPIPagination( request, course_stats_response["page"], - ( - matched_users_pages - if username_search_string - else course_stats_response["num_pages"] - ), - ( - matched_users_count - if username_search_string - else course_stats_response["count"] - ), - ) - return paginator.get_paginated_response( - { - "results": serializer.data, - } + matched_users_pages if username_search_string else course_stats_response["num_pages"], + matched_users_count if username_search_string else course_stats_response["count"], ) + return paginator.get_paginated_response({ + "results": serializer.data, + }) def get_users_without_stats( - username_search_string, course_key, page_number, page_size, request, is_privileged + username_search_string, + course_key, + page_number, + page_size, + request, + is_privileged ): """ This return users with no user stats. This function will be deprecated when this ticket DOS-3414 is resolved """ if username_search_string: - comma_separated_usernames, matched_users_count, matched_users_pages = ( - get_usernames_from_search_string( - course_key, username_search_string, page_number, page_size - ) + comma_separated_usernames, matched_users_count, matched_users_pages = get_usernames_from_search_string( + course_key, username_search_string, page_number, page_size ) if not comma_separated_usernames: - return DiscussionAPIPagination(request, 0, 1).get_paginated_response( - { - "results": [], - } - ) + return DiscussionAPIPagination(request, 0, 1).get_paginated_response({ + "results": [], + }) else: - comma_separated_usernames, matched_users_count, matched_users_pages = ( - get_usernames_for_course(course_key, page_number, page_size) + comma_separated_usernames, matched_users_count, matched_users_pages = get_usernames_for_course( + course_key, page_number, page_size ) if comma_separated_usernames: - updated_course_stats = add_stats_for_users_with_null_values( - [], comma_separated_usernames - ) + updated_course_stats = add_stats_for_users_with_null_values([], comma_separated_usernames) - serializer = UserStatsSerializer( - updated_course_stats, context={"is_privileged": is_privileged}, many=True - ) + serializer = UserStatsSerializer(updated_course_stats, context={"is_privileged": is_privileged}, many=True) paginator = DiscussionAPIPagination( request, page_number, matched_users_pages, matched_users_count, ) - return paginator.get_paginated_response( - { - "results": serializer.data, - } - ) + return paginator.get_paginated_response({ + "results": serializer.data, + }) def add_stats_for_users_with_null_values(course_stats, users_in_course): """ Update users stats for users with no discussion stats available in course """ - users_returned_from_api = [user["username"] for user in course_stats] - user_list = users_in_course.split(",") + users_returned_from_api = [user['username'] for user in course_stats] + user_list = users_in_course.split(',') users_with_no_discussion_content = set(user_list) ^ set(users_returned_from_api) updated_course_stats = course_stats for user in users_with_no_discussion_content: - updated_course_stats.append( - { - "username": user, - "threads": None, - "replies": None, - "responses": None, - "active_flags": None, - "inactive_flags": None, - } - ) - updated_course_stats = sorted( - updated_course_stats, key=lambda d: len(d["username"]) - ) + updated_course_stats.append({ + 'username': user, + 'threads': None, + 'replies': None, + 'responses': None, + 'active_flags': None, + 'inactive_flags': None, + }) + updated_course_stats = sorted(updated_course_stats, key=lambda d: len(d['username'])) return updated_course_stats - - -def _get_user_label_function(course_staff_user_ids, moderator_user_ids, ta_user_ids): - """ - Create and return a function that determines user labels based on role. - - Args: - course_staff_user_ids: List of user IDs for course staff - moderator_user_ids: List of user IDs for moderators - ta_user_ids: List of user IDs for TAs - - Returns: - A function that takes a user_id and returns the appropriate label or None - """ - - def get_user_label(user_id): - """Get role label for a user ID.""" - try: - user_id_int = int(user_id) - if user_id_int in course_staff_user_ids: - return "Staff" - elif user_id_int in moderator_user_ids: - return "Moderator" - elif user_id_int in ta_user_ids: - return "Community TA" - except (ValueError, TypeError): - # If user_id has any issues, there's no label to return - pass - return None - - return get_user_label - - -def _process_deleted_thread(thread_data, get_user_label_fn, usernames_set): - """ - Process a single deleted thread into the standardized content item format. - - Args: - thread_data: Raw thread data from forum API - get_user_label_fn: Function to get user labels by user ID - usernames_set: Set to collect usernames for profile image fetch (modified in-place) - - Returns: - dict: Formatted content item for the thread - """ - author_username = thread_data.get("author_username", "") - deleted_by_id = thread_data.get("deleted_by") - deleted_by_username = None - - # Get deleted_by username - if deleted_by_id: - try: - deleted_user = User.objects.get(id=int(deleted_by_id)) - deleted_by_username = deleted_user.username - usernames_set.add(deleted_by_username) - except (User.DoesNotExist, ValueError): - # If user not found or invalid ID, skip setting deleted fields - pass - - if author_username: - usernames_set.add(author_username) - - # Strip HTML tags from preview - body_text = thread_data.get("body", "") - preview_text = strip_tags(body_text)[:100] if body_text else "" - - thread_id = thread_data.get("_id", thread_data.get("id")) - return { - "id": str(thread_id) + "-thread", - "type": "thread", - "title": thread_data.get("title", ""), - "body": body_text, - "preview_body": preview_text, - "course_id": thread_data.get("course_id", ""), - "author": author_username, - "author_id": thread_data.get("author_id", ""), - "author_label": get_user_label_fn(thread_data.get("author_id")), - "commentable_id": thread_data.get("commentable_id", ""), - "created_at": thread_data.get("created_at"), - "updated_at": thread_data.get("updated_at"), - "is_deleted": True, - "deleted_at": thread_data.get("deleted_at"), - "deleted_by": deleted_by_username, - "deleted_by_label": get_user_label_fn(deleted_by_id) if deleted_by_id else None, - "thread_type": thread_data.get("thread_type", "discussion"), - "anonymous": thread_data.get("anonymous", False), - "anonymous_to_peers": thread_data.get("anonymous_to_peers", False), - "vote_count": thread_data.get("vote_count", 0), - "comment_count": thread_data.get("comment_count", 0), - } - - -def _process_deleted_comment(comment_data, get_user_label_fn, usernames_set): - """ - Process a single deleted comment into the standardized content item format. - - Args: - comment_data: Raw comment data from forum API - get_user_label_fn: Function to get user labels by user ID - usernames_set: Set to collect usernames for profile image fetch (modified in-place) - - Returns: - dict: Formatted content item for the comment - """ - author_username = comment_data.get("author_username", "") - deleted_by_id = comment_data.get("deleted_by") - deleted_by_username = None - - # Get deleted_by username - if deleted_by_id: - try: - deleted_user = User.objects.get(id=int(deleted_by_id)) - deleted_by_username = deleted_user.username - usernames_set.add(deleted_by_username) - except (User.DoesNotExist, ValueError): - # If user not found or invalid ID, skip setting deleted fields - pass - - if author_username: - usernames_set.add(author_username) - - # Determine if this is a response (depth=0) or comment (depth>0) - depth = comment_data.get("depth", 0) - comment_type = "response" if depth == 0 else "comment" - - # Get parent thread title for context - thread_id = comment_data.get("comment_thread_id", "") - thread_title = "" - if thread_id: - try: - parent_thread = Thread(id=thread_id).retrieve() - thread_title = parent_thread.get("title", "") - except Exception: # pylint: disable=broad-exception-caught - pass - - # Strip HTML tags from preview - body_text = comment_data.get("body", "") - preview_text = strip_tags(body_text)[:100] if body_text else "" - - comment_id = comment_data.get("_id", comment_data.get("id")) - return { - "id": str(comment_id) + "-comment", - "type": comment_type, - "body": body_text, - "preview_body": preview_text, - "title": thread_title, # Use parent thread title for comments/responses - "course_id": comment_data.get("course_id", ""), - "author": author_username, - "author_id": comment_data.get("author_id", ""), - "author_label": get_user_label_fn(comment_data.get("author_id")), - "comment_thread_id": str(thread_id), - "thread_title": thread_title, - "parent_id": ( - str(comment_data.get("parent_id", "")) - if comment_data.get("parent_id") - else None - ), - "created_at": comment_data.get("created_at"), - "updated_at": comment_data.get("updated_at"), - "is_deleted": True, - "deleted_at": comment_data.get("deleted_at"), - "deleted_by": deleted_by_username, - "deleted_by_label": get_user_label_fn(deleted_by_id) if deleted_by_id else None, - "depth": depth, - "anonymous": comment_data.get("anonymous", False), - "anonymous_to_peers": comment_data.get("anonymous_to_peers", False), - "endorsed": comment_data.get("endorsed", False), - "vote_count": comment_data.get("vote_count", 0), - } - - -def _add_user_profiles_to_content(deleted_content, usernames_set, request): - """ - Fetch user profile images and add them to each content item. - - Args: - deleted_content: List of content items (modified in-place) - usernames_set: Set of usernames to fetch profile images for - request: Django request object for getting profile images - """ - # Add profile images for all users - username_profile_dict = _get_user_profile_dict( - request, usernames=",".join(usernames_set) - ) - - # Add users dict with profile images to each item - for item in deleted_content: - users_dict = {} - - # Add author profile - author_username = item.get("author") - if author_username and author_username in username_profile_dict: - users_dict[author_username] = _user_profile( - username_profile_dict[author_username] - ) - - # Add deleted_by profile - deleted_by_username = item.get("deleted_by") - if deleted_by_username and deleted_by_username in username_profile_dict: - users_dict[deleted_by_username] = _user_profile( - username_profile_dict[deleted_by_username] - ) - - item["users"] = users_dict - - -def get_deleted_content_for_course( - request, course_id, content_type=None, page=1, per_page=20, author_id=None -): - """ - Retrieve all deleted content (threads, comments) for a course. - - Args: - request: The django request object for getting user profile images - course_id (str): Course identifier - content_type (str, optional): Filter by 'thread' or 'comment'. If None, returns all types. - page (int): Page number for pagination (1-based) - per_page (int): Number of items per page - author_id (str, optional): Filter by author ID - - Returns: - dict: Paginated results with deleted content including author labels and profile images - """ - - import math - - from lms.djangoapps.discussion.rest_api.utils import ( - get_course_staff_users_list, - get_course_ta_users_list, - get_moderator_users_list, - ) - - try: - # Get course and user role information for labels - course_key = CourseKey.from_string(course_id) - course = _get_course(course_key, request.user) - - course_staff_user_ids = get_course_staff_users_list(course.id) - moderator_user_ids = get_moderator_users_list(course.id) - ta_user_ids = get_course_ta_users_list(course.id) - - # Get user label function - get_user_label = _get_user_label_function( - course_staff_user_ids, moderator_user_ids, ta_user_ids - ) - - # Build query parameters for forum API - query_params = { - "course_id": course_id, - "is_deleted": True, # Only get deleted content - "page": page, - "per_page": per_page, - } - - if author_id: - query_params["author_id"] = author_id - - deleted_content = [] - total_count = 0 - usernames_set = set() # Track all usernames for profile image fetch - - # Get deleted threads - if content_type is None or content_type == "thread": - try: - deleted_threads = forum_api.get_deleted_threads_for_course( - course_id=course_id, - page=page if content_type == "thread" else 1, - per_page=per_page if content_type == "thread" else 1000, - author_id=author_id, - ) - for thread_data in deleted_threads.get("threads", []): - content_item = _process_deleted_thread( - thread_data, get_user_label, usernames_set - ) - deleted_content.append(content_item) - - if content_type == "thread": - total_count = deleted_threads.get( - "total_count", len(deleted_content) - ) - except Exception as e: # pylint: disable=broad-exception-caught - log.warning( - "Failed to get deleted threads for course %s: %s", course_id, e - ) - - # Get deleted comments - if content_type is None or content_type == "comment": - try: - deleted_comments = forum_api.get_deleted_comments_for_course( - course_id=course_id, - page=page if content_type == "comment" else 1, - per_page=per_page if content_type == "comment" else 1000, - author_id=author_id, - ) - for comment_data in deleted_comments.get("comments", []): - content_item = _process_deleted_comment( - comment_data, get_user_label, usernames_set - ) - deleted_content.append(content_item) - - if content_type == "comment": - total_count = deleted_comments.get( - "total_count", len(deleted_content) - ) - except Exception as e: # pylint: disable=broad-exception-caught - log.warning( - "Failed to get deleted comments for course %s: %s", course_id, e - ) - - # If getting all content types, handle pagination differently - if content_type is None: - total_count = len(deleted_content) - # Sort by deletion date (most recent first) - deleted_content.sort(key=lambda x: x.get("deleted_at", ""), reverse=True) - - # Apply pagination to combined results - start_idx = (page - 1) * per_page - end_idx = start_idx + per_page - deleted_content = deleted_content[start_idx:end_idx] - - # Add profile images for all users - _add_user_profiles_to_content(deleted_content, usernames_set, request) - - # Calculate pagination info - num_pages = math.ceil(total_count / per_page) if total_count > 0 else 1 - - return { - "results": deleted_content, - "pagination": { - "next": None, # Can be computed if needed - "previous": None, # Can be computed if needed - "count": total_count, - "num_pages": num_pages, - }, - } - - except Exception as e: - log.exception("Error getting deleted content for course %s: %s", course_id, e) - raise diff --git a/lms/djangoapps/discussion/rest_api/forms.py b/lms/djangoapps/discussion/rest_api/forms.py index f37543723792..8cc7127645b2 100644 --- a/lms/djangoapps/discussion/rest_api/forms.py +++ b/lms/djangoapps/discussion/rest_api/forms.py @@ -1,7 +1,6 @@ """ Discussion API forms """ - import urllib.parse from django.core.exceptions import ValidationError @@ -23,15 +22,13 @@ class UserOrdering(TextChoices): - BY_ACTIVITY = "activity" - BY_FLAGS = "flagged" - BY_RECENT_ACTIVITY = "recency" - BY_DELETED = "deleted" + BY_ACTIVITY = 'activity' + BY_FLAGS = 'flagged' + BY_RECENT_ACTIVITY = 'recency' class _PaginationForm(Form): """A form that includes pagination fields""" - page = IntegerField(required=False, min_value=1) page_size = IntegerField(required=False, min_value=1) @@ -48,7 +45,6 @@ class ThreadListGetForm(_PaginationForm): """ A form to validate query parameters in the thread list retrieval endpoint """ - EXCLUSIVE_PARAMS = ["topic_id", "text_search", "following"] course_id = CharField() @@ -62,22 +58,17 @@ class ThreadListGetForm(_PaginationForm): ) count_flagged = ExtendedNullBooleanField(required=False) flagged = ExtendedNullBooleanField(required=False) - show_deleted = ExtendedNullBooleanField(required=False) view = ChoiceField( - choices=[ - (choice, choice) for choice in ["unread", "unanswered", "unresponded"] - ], + choices=[(choice, choice) for choice in ["unread", "unanswered", "unresponded"]], required=False, ) order_by = ChoiceField( - choices=[ - (choice, choice) - for choice in ["last_activity_at", "comment_count", "vote_count"] - ], - required=False, + choices=[(choice, choice) for choice in ["last_activity_at", "comment_count", "vote_count"]], + required=False ) order_direction = ChoiceField( - choices=[(choice, choice) for choice in ["desc"]], required=False + choices=[(choice, choice) for choice in ["desc"]], + required=False ) requested_fields = MultiValueField(required=False) @@ -94,16 +85,14 @@ def clean_course_id(self): value = self.cleaned_data["course_id"] try: return CourseLocator.from_string(value) - except InvalidKeyError as e: - raise ValidationError(f"'{value}' is not a valid course id") from e + except InvalidKeyError: + raise ValidationError(f"'{value}' is not a valid course id") # lint-amnesty, pylint: disable=raise-missing-from def clean_following(self): """Validate following""" value = self.cleaned_data["following"] if value is False: # lint-amnesty, pylint: disable=no-else-raise - raise ValidationError( - "The value of the 'following' parameter must be true." - ) + raise ValidationError("The value of the 'following' parameter must be true.") else: return value @@ -126,7 +115,6 @@ class ThreadActionsForm(Form): A form to handle fields in thread creation/update that require separate interactions with the comments service. """ - following = BooleanField(required=False) voted = BooleanField(required=False) abuse_flagged = BooleanField(required=False) @@ -138,20 +126,17 @@ class CommentListGetForm(_PaginationForm): """ A form to validate query parameters in the comment list retrieval endpoint """ - thread_id = CharField() flagged = BooleanField(required=False) endorsed = ExtendedNullBooleanField(required=False) requested_fields = MultiValueField(required=False) merge_question_type_responses = BooleanField(required=False) - show_deleted = ExtendedNullBooleanField(required=False) class UserCommentListGetForm(_PaginationForm): """ A form to validate query parameters in the comment list retrieval endpoint """ - course_id = CharField() flagged = BooleanField(required=False) requested_fields = MultiValueField(required=False) @@ -161,8 +146,8 @@ def clean_course_id(self): value = self.cleaned_data["course_id"] try: return CourseLocator.from_string(value) - except InvalidKeyError as e: - raise ValidationError(f"'{value}' is not a valid course id") from e + except InvalidKeyError: + raise ValidationError(f"'{value}' is not a valid course id") # lint-amnesty, pylint: disable=raise-missing-from class CommentActionsForm(Form): @@ -170,7 +155,6 @@ class CommentActionsForm(Form): A form to handle fields in comment creation/update that require separate interactions with the comments service. """ - voted = BooleanField(required=False) abuse_flagged = BooleanField(required=False) @@ -179,7 +163,6 @@ class CommentGetForm(_PaginationForm): """ A form to validate query parameters in the comment retrieval endpoint """ - requested_fields = MultiValueField(required=False) @@ -187,34 +170,28 @@ class CourseDiscussionSettingsForm(Form): """ A form to validate the fields in the course discussion settings requests. """ - course_id = CharField() def __init__(self, *args, **kwargs): - self.request_user = kwargs.pop("request_user") + self.request_user = kwargs.pop('request_user') super().__init__(*args, **kwargs) def clean_course_id(self): """Validate the 'course_id' value""" - course_id = self.cleaned_data["course_id"] + course_id = self.cleaned_data['course_id'] try: course_key = CourseKey.from_string(course_id) - self.cleaned_data["course"] = get_course_with_access( - self.request_user, "load", course_key - ) - self.cleaned_data["course_key"] = course_key + self.cleaned_data['course'] = get_course_with_access(self.request_user, 'load', course_key) + self.cleaned_data['course_key'] = course_key return course_id - except InvalidKeyError as e: - raise ValidationError( - f"'{str(course_id)}' is not a valid course key" - ) from e + except InvalidKeyError: + raise ValidationError(f"'{str(course_id)}' is not a valid course key") # lint-amnesty, pylint: disable=raise-missing-from class CourseDiscussionRolesForm(CourseDiscussionSettingsForm): """ A form to validate the fields in the course discussion roles requests. """ - ROLE_CHOICES = ( (FORUM_ROLE_MODERATOR, FORUM_ROLE_MODERATOR), (FORUM_ROLE_COMMUNITY_TA, FORUM_ROLE_MODERATOR), @@ -222,20 +199,20 @@ class CourseDiscussionRolesForm(CourseDiscussionSettingsForm): ) rolename = ChoiceField( choices=ROLE_CHOICES, - error_messages={"invalid_choice": "Role '%(value)s' does not exist"}, + error_messages={"invalid_choice": "Role '%(value)s' does not exist"} ) def clean_rolename(self): """Validate the 'rolename' value.""" - rolename = urllib.parse.unquote(self.cleaned_data.get("rolename")) - course_id = self.cleaned_data.get("course_key") + rolename = urllib.parse.unquote(self.cleaned_data.get('rolename')) + course_id = self.cleaned_data.get('course_key') if course_id and rolename: try: role = Role.objects.get(name=rolename, course_id=course_id) except Role.DoesNotExist as err: raise ValidationError(f"Role '{rolename}' does not exist") from err - self.cleaned_data["role"] = role + self.cleaned_data['role'] = role return rolename @@ -243,17 +220,15 @@ class TopicListGetForm(Form): """ Form for the topics API get query parameters. """ - topic_id = CharField(required=False) order_by = ChoiceField(choices=TopicOrdering.choices, required=False) def clean_topic_id(self): topic_ids = self.cleaned_data.get("topic_id", None) - return set(topic_ids.strip(",").split(",")) if topic_ids else None + return set(topic_ids.strip(',').split(',')) if topic_ids else None class CourseActivityStatsForm(_PaginationForm): """Form for validating course activity stats API query parameters""" - order_by = ChoiceField(choices=UserOrdering.choices, required=False) username = CharField(required=False) diff --git a/lms/djangoapps/discussion/rest_api/serializers.py b/lms/djangoapps/discussion/rest_api/serializers.py index 902a433dac3b..8a7ab16e0903 100644 --- a/lms/djangoapps/discussion/rest_api/serializers.py +++ b/lms/djangoapps/discussion/rest_api/serializers.py @@ -1,13 +1,13 @@ """ Discussion API serializers """ - import html import re + +from bs4 import BeautifulSoup from typing import Dict from urllib.parse import urlencode, urlunparse -from bs4 import BeautifulSoup from django.conf import settings from django.contrib.auth import get_user_model from django.core.exceptions import ObjectDoesNotExist, ValidationError @@ -18,12 +18,8 @@ from common.djangoapps.student.models import get_user_by_username_or_email from common.djangoapps.student.roles import GlobalStaff -from lms.djangoapps.discussion.django_comment_client.base.views import ( - track_comment_edited_event, - track_forum_response_mark_event, - track_thread_edited_event, - track_thread_lock_unlock_event, -) +from lms.djangoapps.discussion.django_comment_client.base.views import track_thread_lock_unlock_event, \ + track_thread_edited_event, track_comment_edited_event, track_forum_response_mark_event from lms.djangoapps.discussion.django_comment_client.utils import ( course_discussion_division_enabled, get_group_id_for_user, @@ -39,23 +35,17 @@ from lms.djangoapps.discussion.rest_api.render import render_body from lms.djangoapps.discussion.rest_api.utils import ( get_course_staff_users_list, - get_course_ta_users_list, get_moderator_users_list, + get_course_ta_users_list, get_user_learner_status, ) from openedx.core.djangoapps.discussions.models import DiscussionTopicLink from openedx.core.djangoapps.discussions.utils import get_group_names_by_id from openedx.core.djangoapps.django_comment_common.comment_client.comment import Comment from openedx.core.djangoapps.django_comment_common.comment_client.thread import Thread -from openedx.core.djangoapps.django_comment_common.comment_client.user import ( - User as CommentClientUser, -) -from openedx.core.djangoapps.django_comment_common.comment_client.utils import ( - CommentClientRequestError, -) -from openedx.core.djangoapps.django_comment_common.models import ( - CourseDiscussionSettings, -) +from openedx.core.djangoapps.django_comment_common.comment_client.user import User as CommentClientUser +from openedx.core.djangoapps.django_comment_common.comment_client.utils import CommentClientRequestError +from openedx.core.djangoapps.django_comment_common.models import CourseDiscussionSettings from openedx.core.djangoapps.user_api.accounts.api import get_profile_images from openedx.core.lib.api.serializers import CourseKeyField @@ -69,7 +59,6 @@ class TopicOrdering(TextChoices): """ Enum for the available options for ordering topics. """ - COURSE_STRUCTURE = "course_structure", "Course Structure" ACTIVITY = "activity", "Activity" NAME = "name", "Name" @@ -84,24 +73,16 @@ def get_context(course, request, thread=None): moderator_user_ids = get_moderator_users_list(course.id) ta_user_ids = get_course_ta_users_list(course.id) requester = request.user - cc_requester = CommentClientUser.from_django_user(requester).retrieve( - course_id=course.id - ) + cc_requester = CommentClientUser.from_django_user(requester).retrieve(course_id=course.id) cc_requester["course_id"] = course.id course_discussion_settings = CourseDiscussionSettings.get(course.id) is_global_staff = GlobalStaff().has_user(requester) - has_moderation_privilege = ( - requester.id in moderator_user_ids - or requester.id in ta_user_ids - or is_global_staff - ) + has_moderation_privilege = requester.id in moderator_user_ids or requester.id in ta_user_ids or is_global_staff return { "course": course, "request": request, "thread": thread, - "discussion_division_enabled": course_discussion_division_enabled( - course_discussion_settings - ), + "discussion_division_enabled": course_discussion_division_enabled(course_discussion_settings), "group_ids_to_names": get_group_names_by_id(course_discussion_settings), "moderator_user_ids": moderator_user_ids, "course_staff_user_ids": course_staff_user_ids, @@ -156,8 +137,8 @@ def _validate_privileged_access(context: Dict) -> bool: Returns: bool: Course exists and the user has privileged access. """ - course = context.get("course", None) - is_requester_privileged = context.get("has_moderation_privilege") + course = context.get('course', None) + is_requester_privileged = context.get('has_moderation_privilege') return course and is_requester_privileged @@ -177,7 +158,7 @@ def filter_spam_urls_from_html(html_string): patterns.append(re.compile(rf"(https?://)?{domain_pattern}", re.IGNORECASE)) for a_tag in soup.find_all("a", href=True): - href = a_tag.get("href") + href = a_tag.get('href') if href: if any(p.search(href) for p in patterns): a_tag.replace_with(a_tag.get_text(strip=True)) @@ -186,7 +167,7 @@ def filter_spam_urls_from_html(html_string): for text_node in soup.find_all(string=True): new_text = text_node for p in patterns: - new_text = p.sub("", new_text) + new_text = p.sub('', new_text) if new_text != text_node: text_node.replace_with(new_text.strip()) is_spam = True @@ -215,14 +196,8 @@ class _ContentSerializer(serializers.Serializer): anonymous = serializers.BooleanField(default=False) anonymous_to_peers = serializers.BooleanField(default=False) last_edit = serializers.SerializerMethodField(required=False) - edit_reason_code = serializers.CharField( - required=False, validators=[validate_edit_reason_code] - ) + edit_reason_code = serializers.CharField(required=False, validators=[validate_edit_reason_code]) edit_by_label = serializers.SerializerMethodField(required=False) - is_deleted = serializers.SerializerMethodField(read_only=True) - deleted_at = serializers.SerializerMethodField(read_only=True) - deleted_by = serializers.SerializerMethodField(read_only=True) - deleted_by_label = serializers.SerializerMethodField(read_only=True) non_updatable_fields = set() @@ -244,10 +219,7 @@ def _is_user_privileged(self, user_id): Returns a boolean indicating whether the given user_id identifies a privileged user. """ - return ( - user_id in self.context["moderator_user_ids"] - or user_id in self.context["ta_user_ids"] - ) + return user_id in self.context["moderator_user_ids"] or user_id in self.context["ta_user_ids"] def _is_anonymous(self, obj): """ @@ -255,12 +227,12 @@ def _is_anonymous(self, obj): the requester. """ user_id = self.context["request"].user.id - is_user_staff = ( - user_id in self.context["moderator_user_ids"] - or user_id in self.context["ta_user_ids"] - ) + is_user_staff = user_id in self.context["moderator_user_ids"] or user_id in self.context["ta_user_ids"] - return obj["anonymous"] or obj["anonymous_to_peers"] and not is_user_staff + return ( + obj["anonymous"] or + obj["anonymous_to_peers"] and not is_user_staff + ) def get_author(self, obj): """ @@ -278,9 +250,10 @@ def _get_user_label(self, user_id): is_ta = user_id in self.context["ta_user_ids"] return ( - "Staff" - if is_staff - else "Moderator" if is_moderator else "Community TA" if is_ta else None + "Staff" if is_staff else + "Moderator" if is_moderator else + "Community TA" if is_ta else + None ) def _get_user_label_from_username(self, username): @@ -330,9 +303,7 @@ def get_rendered_body(self, obj): """ if self._rendered_body is None: self._rendered_body = render_body(obj["body"]) - self._rendered_body, is_spam = filter_spam_urls_from_html( - self._rendered_body - ) + self._rendered_body, is_spam = filter_spam_urls_from_html(self._rendered_body) if is_spam and settings.CONTENT_FOR_SPAM_POSTS: self._rendered_body = settings.CONTENT_FOR_SPAM_POSTS return self._rendered_body @@ -344,9 +315,8 @@ def get_abuse_flagged(self, obj): """ total_abuse_flaggers = len(obj.get("abuse_flaggers", [])) return ( - self.context["has_moderation_privilege"] - and total_abuse_flaggers > 0 - or self.context["cc_requester"]["id"] in obj.get("abuse_flaggers", []) + self.context["has_moderation_privilege"] and total_abuse_flaggers > 0 or + self.context["cc_requester"]["id"] in obj.get("abuse_flaggers", []) ) def get_voted(self, obj): @@ -379,7 +349,7 @@ def get_last_edit(self, obj): Returns information about the last edit for this content for privileged users. """ - is_user_author = str(obj["user_id"]) == str(self.context["request"].user.id) + is_user_author = str(obj['user_id']) == str(self.context['request'].user.id) if not (_validate_privileged_access(self.context) or is_user_author): return None edit_history = obj.get("edit_history") @@ -395,57 +365,12 @@ def get_edit_by_label(self, obj): """ Returns the role label for the last edit user. """ - is_user_author = str(obj["user_id"]) == str(self.context["request"].user.id) + is_user_author = str(obj['user_id']) == str(self.context['request'].user.id) is_user_privileged = _validate_privileged_access(self.context) edit_history = obj.get("edit_history") if (is_user_author or is_user_privileged) and edit_history: last_edit = edit_history[-1] - return self._get_user_label_from_username(last_edit.get("editor_username")) - - def get_is_deleted(self, obj): - """ - Returns the is_deleted status for privileged users only. - """ - if not _validate_privileged_access(self.context): - return None - return obj.get("is_deleted", False) - - def get_deleted_at(self, obj): - """ - Returns the deletion timestamp for privileged users only. - """ - if not _validate_privileged_access(self.context): - return None - return obj.get("deleted_at") - - def get_deleted_by(self, obj): - """ - Returns the username of the user who deleted this content for privileged users only. - """ - if not _validate_privileged_access(self.context): - return None - deleted_by_id = obj.get("deleted_by") - if deleted_by_id: - try: - user = User.objects.get(id=int(deleted_by_id)) - return user.username - except (User.DoesNotExist, ValueError): - return None - return None - - def get_deleted_by_label(self, obj): - """ - Returns the role label for the user who deleted this content for privileged users only. - """ - if not _validate_privileged_access(self.context): - return None - deleted_by_id = obj.get("deleted_by") - if deleted_by_id: - try: - return self._get_user_label(int(deleted_by_id)) - except (ValueError, TypeError): - return None - return None + return self._get_user_label_from_username(last_edit.get('editor_username')) class ThreadSerializer(_ContentSerializer): @@ -456,15 +381,13 @@ class ThreadSerializer(_ContentSerializer): not had retrieve() called, because of the interaction between DRF's attempts at introspection and Thread's __getattr__. """ - course_id = serializers.CharField() - topic_id = serializers.CharField( - source="commentable_id", validators=[validate_not_blank] - ) + topic_id = serializers.CharField(source="commentable_id", validators=[validate_not_blank]) group_id = serializers.IntegerField(required=False, allow_null=True) group_name = serializers.SerializerMethodField() type = serializers.ChoiceField( - source="thread_type", choices=[(val, val) for val in ["discussion", "question"]] + source="thread_type", + choices=[(val, val) for val in ["discussion", "question"]] ) preview_body = serializers.SerializerMethodField() abuse_flagged_count = serializers.SerializerMethodField(required=False) @@ -479,12 +402,8 @@ class ThreadSerializer(_ContentSerializer): non_endorsed_comment_list_url = serializers.SerializerMethodField() read = serializers.BooleanField(required=False) has_endorsed = serializers.BooleanField(source="endorsed", read_only=True) - response_count = serializers.IntegerField( - source="resp_total", read_only=True, required=False - ) - close_reason_code = serializers.CharField( - required=False, validators=[validate_close_reason_code] - ) + response_count = serializers.IntegerField(source="resp_total", read_only=True, required=False) + close_reason_code = serializers.CharField(required=False, validators=[validate_close_reason_code]) close_reason = serializers.SerializerMethodField() closed_by = serializers.SerializerMethodField() closed_by_label = serializers.SerializerMethodField(required=False) @@ -530,8 +449,9 @@ def get_comment_list_url(self, obj, endorsed=None): Returns the URL to retrieve the thread's comments, optionally including the endorsed query parameter. """ - if (obj["thread_type"] == "question" and endorsed is None) or ( - obj["thread_type"] == "discussion" and endorsed is not None + if ( + (obj["thread_type"] == "question" and endorsed is None) or + (obj["thread_type"] == "discussion" and endorsed is not None) ): return None path = reverse("comment-list") @@ -575,17 +495,13 @@ def get_preview_body(self, obj): """ Returns a cleaned version of the thread's body to display in a preview capacity. """ - return ( - strip_tags(self.get_rendered_body(obj)) - .replace("\n", " ") - .replace(" ", " ") - ) + return strip_tags(self.get_rendered_body(obj)).replace('\n', ' ').replace(' ', ' ') def get_close_reason(self, obj): """ Returns the reason for which the thread was closed. """ - is_user_author = str(obj["user_id"]) == str(self.context["request"].user.id) + is_user_author = str(obj['user_id']) == str(self.context['request'].user.id) if not (_validate_privileged_access(self.context) or is_user_author): return None reason_code = obj.get("close_reason_code") @@ -596,7 +512,7 @@ def get_closed_by(self, obj): Returns the username of the moderator who closed this thread, only to other privileged users and author. """ - is_user_author = str(obj["user_id"]) == str(self.context["request"].user.id) + is_user_author = str(obj['user_id']) == str(self.context['request'].user.id) if _validate_privileged_access(self.context) or is_user_author: return obj.get("closed_by") @@ -604,7 +520,7 @@ def get_closed_by_label(self, obj): """ Returns the role label for the user who closed the post. """ - is_user_author = str(obj["user_id"]) == str(self.context["request"].user.id) + is_user_author = str(obj['user_id']) == str(self.context['request'].user.id) if is_user_author or _validate_privileged_access(self.context): return self._get_user_label_from_username(obj.get("closed_by")) @@ -619,31 +535,18 @@ def update(self, instance, validated_data): requesting_user_id = self.context["cc_requester"]["id"] if key == "closed" and val: instance["closing_user_id"] = requesting_user_id - track_thread_lock_unlock_event( - self.context["request"], - self.context["course"], - instance, - validated_data.get("close_reason_code"), - ) + track_thread_lock_unlock_event(self.context['request'], self.context['course'], + instance, validated_data.get('close_reason_code')) if key == "closed" and not val: instance["closing_user_id"] = requesting_user_id - track_thread_lock_unlock_event( - self.context["request"], - self.context["course"], - instance, - validated_data.get("close_reason_code"), - locked=False, - ) + track_thread_lock_unlock_event(self.context['request'], self.context['course'], + instance, validated_data.get('close_reason_code'), locked=False) if key == "body" and val: instance["editing_user_id"] = requesting_user_id - track_thread_edited_event( - self.context["request"], - self.context["course"], - instance, - validated_data.get("edit_reason_code"), - ) + track_thread_edited_event(self.context['request'], self.context['course'], + instance, validated_data.get('edit_reason_code')) instance.save() return instance @@ -656,7 +559,6 @@ class CommentSerializer(_ContentSerializer): not had retrieve() called, because of the interaction between DRF's attempts at introspection and Comment's __getattr__. """ - thread_id = serializers.CharField() parent_id = serializers.CharField(required=False, allow_null=True) endorsed = serializers.BooleanField(required=False) @@ -671,7 +573,7 @@ class CommentSerializer(_ContentSerializer): non_updatable_fields = NON_UPDATABLE_COMMENT_FIELDS def __init__(self, *args, **kwargs): - remove_fields = kwargs.pop("remove_fields", None) + remove_fields = kwargs.pop('remove_fields', None) super().__init__(*args, **kwargs) if remove_fields: @@ -693,8 +595,8 @@ def get_endorsed_by(self, obj): # Avoid revealing the identity of an anonymous non-staff question # author who has endorsed a comment in the thread if not ( - self._is_anonymous(self.context["thread"]) - and not self._is_user_privileged(endorser_id) + self._is_anonymous(self.context["thread"]) and + not self._is_user_privileged(endorser_id) ): return User.objects.get(id=endorser_id).username return None @@ -736,7 +638,7 @@ def to_representation(self, data): # Django Rest Framework v3 no longer includes None values # in the representation. To maintain the previous behavior, # we do this manually instead. - if "parent_id" not in data: + if 'parent_id' not in data: data["parent_id"] = None return data @@ -778,7 +680,7 @@ def create(self, validated_data): comment = Comment( course_id=self.context["thread"]["course_id"], user_id=self.context["cc_requester"]["id"], - **validated_data, + **validated_data ) comment.save() return comment @@ -791,18 +693,12 @@ def update(self, instance, validated_data): # endorsement_user_id on update requesting_user_id = self.context["cc_requester"]["id"] if key == "endorsed": - track_forum_response_mark_event( - self.context["request"], self.context["course"], instance, val - ) + track_forum_response_mark_event(self.context['request'], self.context['course'], instance, val) instance["endorsement_user_id"] = requesting_user_id if key == "body" and val: instance["editing_user_id"] = requesting_user_id - track_comment_edited_event( - self.context["request"], - self.context["course"], - instance, - validated_data.get("edit_reason_code"), - ) + track_comment_edited_event(self.context['request'], self.context['course'], + instance, validated_data.get('edit_reason_code')) instance.save() return instance @@ -812,7 +708,6 @@ class DiscussionTopicSerializer(serializers.Serializer): """ Serializer for DiscussionTopic """ - id = serializers.CharField(read_only=True) # pylint: disable=invalid-name name = serializers.CharField(read_only=True) thread_list_url = serializers.CharField(read_only=True) @@ -842,11 +737,10 @@ class DiscussionTopicSerializerV2(serializers.Serializer): """ Serializer for new style topics. """ - id = serializers.CharField( # pylint: disable=invalid-name read_only=True, source="external_id", - help_text="Provider-specific unique id for the topic", + help_text="Provider-specific unique id for the topic" ) usage_key = serializers.CharField( read_only=True, @@ -870,13 +764,10 @@ def get_thread_counts(self, obj: DiscussionTopicLink) -> Dict[str, int]: """ Get thread counts from provided context """ - return self.context["thread_counts"].get( - obj.external_id, - { - "discussion": 0, - "question": 0, - }, - ) + return self.context['thread_counts'].get(obj.external_id, { + "discussion": 0, + "question": 0, + }) class DiscussionRolesSerializer(serializers.Serializer): @@ -884,7 +775,10 @@ class DiscussionRolesSerializer(serializers.Serializer): Serializer for course discussion roles. """ - ACTION_CHOICES = (("allow", "allow"), ("revoke", "revoke")) + ACTION_CHOICES = ( + ('allow', 'allow'), + ('revoke', 'revoke') + ) action = serializers.ChoiceField(ACTION_CHOICES) user_id = serializers.CharField() @@ -905,16 +799,14 @@ def validate_user_id(self, user_id): self.user = get_user_by_username_or_email(user_id) return user_id except User.DoesNotExist as err: - raise ValidationError( - f"'{user_id}' is not a valid student identifier" - ) from err + raise ValidationError(f"'{user_id}' is not a valid student identifier") from err def validate(self, attrs): """Validate the data at an object level.""" # Store the user object to avoid fetching it again. - if hasattr(self, "user"): - attrs["user"] = self.user + if hasattr(self, 'user'): + attrs['user'] = self.user return attrs def create(self, validated_data): @@ -932,7 +824,6 @@ class DiscussionRolesMemberSerializer(serializers.Serializer): """ Serializer for course discussion roles member data. """ - username = serializers.CharField() email = serializers.EmailField() first_name = serializers.CharField() @@ -941,7 +832,7 @@ class DiscussionRolesMemberSerializer(serializers.Serializer): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) - self.course_discussion_settings = self.context["course_discussion_settings"] + self.course_discussion_settings = self.context['course_discussion_settings'] def get_group_name(self, instance): """Return the group name of the user.""" @@ -964,7 +855,6 @@ class DiscussionRolesListSerializer(serializers.Serializer): """ Serializer for course discussion roles member list. """ - course_id = serializers.CharField() results = serializers.SerializerMethodField() division_scheme = serializers.SerializerMethodField() @@ -972,17 +862,15 @@ class DiscussionRolesListSerializer(serializers.Serializer): def get_results(self, obj): """Return the nested serializer data representing a list of member users.""" context = { - "course_id": obj["course_id"], - "course_discussion_settings": self.context["course_discussion_settings"], + 'course_id': obj['course_id'], + 'course_discussion_settings': self.context['course_discussion_settings'] } - serializer = DiscussionRolesMemberSerializer( - obj["users"], context=context, many=True - ) + serializer = DiscussionRolesMemberSerializer(obj['users'], context=context, many=True) return serializer.data def get_division_scheme(self, obj): # pylint: disable=unused-argument """Return the division scheme for the course.""" - return self.context["course_discussion_settings"].division_scheme + return self.context['course_discussion_settings'].division_scheme def create(self, validated_data): """ @@ -999,13 +887,9 @@ class UserStatsSerializer(serializers.Serializer): """ Serializer for course user stats. """ - threads = serializers.IntegerField() replies = serializers.IntegerField() responses = serializers.IntegerField() - deleted_threads = serializers.IntegerField(required=False, default=0) - deleted_replies = serializers.IntegerField(required=False, default=0) - deleted_responses = serializers.IntegerField(required=False, default=0) active_flags = serializers.IntegerField() inactive_flags = serializers.IntegerField() username = serializers.CharField() @@ -1023,36 +907,27 @@ class BlackoutDateSerializer(serializers.Serializer): """ Serializer for blackout dates. """ - - start = serializers.DateTimeField( - help_text="The ISO 8601 timestamp for the start of the blackout period" - ) - end = serializers.DateTimeField( - help_text="The ISO 8601 timestamp for the end of the blackout period" - ) + start = serializers.DateTimeField(help_text="The ISO 8601 timestamp for the start of the blackout period") + end = serializers.DateTimeField(help_text="The ISO 8601 timestamp for the end of the blackout period") class ReasonCodeSeralizer(serializers.Serializer): """ Serializer for reason codes. """ - code = serializers.CharField(help_text="A code for the an edit or close reason") - label = serializers.CharField( - help_text="A user-friendly name text for the close or edit reason" - ) + label = serializers.CharField(help_text="A user-friendly name text for the close or edit reason") class CourseMetadataSerailizer(serializers.Serializer): """ Serializer for course metadata. """ - id = CourseKeyField(help_text="The identifier of the course") blackouts = serializers.ListField( child=BlackoutDateSerializer(), help_text="A list of objects representing blackout periods " - "(during which discussions are read-only except for privileged users).", + "(during which discussions are read-only except for privileged users)." ) thread_list_url = serializers.URLField( help_text="The URL of the list of all threads in the course.", @@ -1060,9 +935,7 @@ class CourseMetadataSerailizer(serializers.Serializer): following_thread_list_url = serializers.URLField( help_text="thread_list_url with parameter following=True", ) - topics_url = serializers.URLField( - help_text="The URL of the topic listing for the course." - ) + topics_url = serializers.URLField(help_text="The URL of the topic listing for the course.") allow_anonymous = serializers.BooleanField( help_text="A boolean indicating whether anonymous posts are allowed or not.", ) diff --git a/lms/djangoapps/discussion/rest_api/tasks.py b/lms/djangoapps/discussion/rest_api/tasks.py index 5773fbbc83b0..cd725a3513dc 100644 --- a/lms/djangoapps/discussion/rest_api/tasks.py +++ b/lms/djangoapps/discussion/rest_api/tasks.py @@ -1,36 +1,32 @@ """ Contain celery tasks """ - import logging from celery import shared_task from django.contrib.auth import get_user_model from edx_django_utils.monitoring import set_code_owner_attribute -from eventtracking import tracker from opaque_keys.edx.locator import CourseKey +from eventtracking import tracker -from common.djangoapps.student.roles import CourseInstructorRole, CourseStaffRole +from common.djangoapps.student.roles import CourseStaffRole, CourseInstructorRole from common.djangoapps.track import segment from lms.djangoapps.courseware.courses import get_course_with_access from lms.djangoapps.discussion.django_comment_client.utils import get_user_role_names -from lms.djangoapps.discussion.rest_api.discussions_notifications import ( - DiscussionNotificationSender, -) +from lms.djangoapps.discussion.rest_api.discussions_notifications import DiscussionNotificationSender from lms.djangoapps.discussion.rest_api.utils import can_user_notify_all_learners from openedx.core.djangoapps.django_comment_common.comment_client import Comment from openedx.core.djangoapps.django_comment_common.comment_client.thread import Thread from openedx.core.djangoapps.notifications.config.waffle import ENABLE_NOTIFICATIONS + User = get_user_model() log = logging.getLogger(__name__) @shared_task @set_code_owner_attribute -def send_thread_created_notification( - thread_id, course_key_str, user_id, notify_all_learners=False -): +def send_thread_created_notification(thread_id, course_key_str, user_id, notify_all_learners=False): """ Send notification when a new thread is created """ @@ -44,21 +40,17 @@ def send_thread_created_notification( is_course_staff = CourseStaffRole(course_key).has_user(user) is_course_admin = CourseInstructorRole(course_key).has_user(user) user_roles = get_user_role_names(user, course_key) - if not can_user_notify_all_learners( - user_roles, is_course_staff, is_course_admin - ): + if not can_user_notify_all_learners(user_roles, is_course_staff, is_course_admin): return - course = get_course_with_access(user, "load", course_key, check_if_enrolled=True) + course = get_course_with_access(user, 'load', course_key, check_if_enrolled=True) notification_sender = DiscussionNotificationSender(thread, course, user) notification_sender.send_new_thread_created_notification(notify_all_learners) @shared_task @set_code_owner_attribute -def send_response_notifications( - thread_id, course_key_str, user_id, comment_id, parent_id=None -): +def send_response_notifications(thread_id, course_key_str, user_id, comment_id, parent_id=None): """ Send notifications to users who are subscribed to the thread. """ @@ -67,10 +59,8 @@ def send_response_notifications( return thread = Thread(id=thread_id).retrieve() user = User.objects.get(id=user_id) - course = get_course_with_access(user, "load", course_key, check_if_enrolled=True) - notification_sender = DiscussionNotificationSender( - thread, course, user, parent_id, comment_id - ) + course = get_course_with_access(user, 'load', course_key, check_if_enrolled=True) + notification_sender = DiscussionNotificationSender(thread, course, user, parent_id, comment_id) notification_sender.send_new_comment_notification() notification_sender.send_new_response_notification() notification_sender.send_new_comment_on_response_notification() @@ -79,9 +69,7 @@ def send_response_notifications( @shared_task @set_code_owner_attribute -def send_response_endorsed_notifications( - thread_id, response_id, course_key_str, endorsed_by -): +def send_response_endorsed_notifications(thread_id, response_id, course_key_str, endorsed_by): """ Send notifications when a response is marked answered/ endorsed """ @@ -92,10 +80,8 @@ def send_response_endorsed_notifications( response = Comment(id=response_id).retrieve() creator = User.objects.get(id=response.user_id) endorser = User.objects.get(id=endorsed_by) - course = get_course_with_access(creator, "load", course_key, check_if_enrolled=True) - notification_sender = DiscussionNotificationSender( - thread, course, creator, comment_id=response_id - ) + course = get_course_with_access(creator, 'load', course_key, check_if_enrolled=True) + notification_sender = DiscussionNotificationSender(thread, course, creator, comment_id=response_id) # skip sending notification to author of thread if they are the same as the author of the response if response.user_id != thread.user_id: # sends notification to author of thread @@ -113,63 +99,15 @@ def delete_course_post_for_user(user_id, username, course_ids, event_data=None): Deletes all posts for user in a course. """ event_data = event_data or {} - log.info( - f"<> Deleting all posts for {username} in course {course_ids}" - ) - # Get triggered_by user_id from event_data for audit trail - deleted_by_user_id = event_data.get("triggered_by_user_id") if event_data else None - threads_deleted = Thread.delete_user_threads( - user_id, course_ids, deleted_by=deleted_by_user_id - ) - comments_deleted = Comment.delete_user_comments( - user_id, course_ids, deleted_by=deleted_by_user_id - ) - log.info( - f"<> Deleted {threads_deleted} posts and {comments_deleted} comments for {username} " - f"in course {course_ids}" - ) - event_data.update( - { - "number_of_posts_deleted": threads_deleted, - "number_of_comments_deleted": comments_deleted, - } - ) - event_name = "edx.discussion.bulk_delete_user_posts" - tracker.emit(event_name, event_data) - segment.track("None", event_name, event_data) - - -@shared_task -@set_code_owner_attribute -def restore_course_post_for_user(user_id, username, course_ids, event_data=None): - """ - Restores all soft-deleted posts for user in a course by setting is_deleted=False. - """ - event_data = event_data or {} - log.info( - "<> Restoring all posts for %s in course %s", username, course_ids - ) - # Get triggered_by user_id from event_data for audit trail - restored_by_user_id = event_data.get("triggered_by_user_id") if event_data else None - threads_restored = Thread.restore_user_deleted_threads( - user_id, course_ids, restored_by=restored_by_user_id - ) - comments_restored = Comment.restore_user_deleted_comments( - user_id, course_ids, restored_by=restored_by_user_id - ) - log.info( - "<> Restored %s posts and %s comments for %s in course %s", - threads_restored, - comments_restored, - username, - course_ids, - ) - event_data.update( - { - "number_of_posts_restored": threads_restored, - "number_of_comments_restored": comments_restored, - } - ) - event_name = "edx.discussion.bulk_restore_user_posts" + log.info(f"<> Deleting all posts for {username} in course {course_ids}") + threads_deleted = Thread.delete_user_threads(user_id, course_ids) + comments_deleted = Comment.delete_user_comments(user_id, course_ids) + log.info(f"<> Deleted {threads_deleted} posts and {comments_deleted} comments for {username} " + f"in course {course_ids}") + event_data.update({ + "number_of_posts_deleted": threads_deleted, + "number_of_comments_deleted": comments_deleted, + }) + event_name = 'edx.discussion.bulk_delete_user_posts' tracker.emit(event_name, event_data) - segment.track("None", event_name, event_data) + segment.track('None', event_name, event_data) diff --git a/lms/djangoapps/discussion/rest_api/tests/test_api_v2.py b/lms/djangoapps/discussion/rest_api/tests/test_api_v2.py index 2fa761b46615..53c12454aec9 100644 --- a/lms/djangoapps/discussion/rest_api/tests/test_api_v2.py +++ b/lms/djangoapps/discussion/rest_api/tests/test_api_v2.py @@ -10,20 +10,34 @@ import random from datetime import datetime, timedelta from unittest import mock +from urllib.parse import parse_qs, urlencode, urlparse, urlunparse import ddt import httpretty import pytest +from django.test import override_settings from django.contrib.auth import get_user_model from django.core.exceptions import ValidationError from django.test.client import RequestFactory +from opaque_keys.edx.keys import CourseKey from opaque_keys.edx.locator import CourseLocator from pytz import UTC from rest_framework.exceptions import PermissionDenied +from xmodule.modulestore import ModuleStoreEnum +from xmodule.modulestore.django import modulestore +from xmodule.modulestore.tests.django_utils import ( + ModuleStoreTestCase, + SharedModuleStoreTestCase, +) +from xmodule.modulestore.tests.factories import CourseFactory, BlockFactory +from xmodule.partitions.partitions import Group, UserPartition + from common.djangoapps.student.tests.factories import ( AdminFactory, + BetaTesterFactory, CourseEnrollmentFactory, + StaffFactory, UserFactory, ) from common.djangoapps.util.testing import UrlResetMixin @@ -31,6 +45,10 @@ from lms.djangoapps.discussion.django_comment_client.tests.utils import ( ForumsEnableMixin, ) +from lms.djangoapps.discussion.tests.utils import ( + make_minimal_cs_comment, + make_minimal_cs_thread, +) from lms.djangoapps.discussion.rest_api import api from lms.djangoapps.discussion.rest_api.api import ( create_comment, @@ -38,9 +56,12 @@ delete_comment, delete_thread, get_comment_list, + get_course, + get_course_topics, get_course_topics_v2, get_thread, get_thread_list, + get_user_comments, update_comment, update_thread, ) @@ -52,19 +73,18 @@ ) from lms.djangoapps.discussion.rest_api.serializers import TopicOrdering from lms.djangoapps.discussion.rest_api.tests.utils import ( + CommentsServiceMockMixin, ForumMockUtilsMixin, make_paginated_api_response, + parsed_body, ) -from lms.djangoapps.discussion.tests.utils import ( - make_minimal_cs_comment, - make_minimal_cs_thread, -) +from openedx.core.djangoapps.course_groups.models import CourseUserGroupPartitionGroup from openedx.core.djangoapps.course_groups.tests.helpers import CohortFactory from openedx.core.djangoapps.discussions.models import ( DiscussionsConfiguration, DiscussionTopicLink, - PostingRestriction, Provider, + PostingRestriction, ) from openedx.core.djangoapps.discussions.tasks import ( update_discussions_settings_from_course_task, @@ -78,13 +98,6 @@ Role, ) from openedx.core.lib.exceptions import CourseNotFoundError, PageNotFoundError -from xmodule.modulestore import ModuleStoreEnum -from xmodule.modulestore.django import modulestore -from xmodule.modulestore.tests.django_utils import ( - ModuleStoreTestCase, - SharedModuleStoreTestCase, -) -from xmodule.modulestore.tests.factories import BlockFactory, CourseFactory User = get_user_model() @@ -261,11 +274,7 @@ def test_basic(self, mock_emit): ) self.register_post_thread_response(cs_thread) with self.assert_signal_sent( - api, - "thread_created", - sender=None, - user=self.user, - exclude_args=("post", "notify_all_learners"), + api, "thread_created", sender=None, user=self.user, exclude_args=("post", "notify_all_learners") ): actual = create_thread(self.request, self.minimal_data) expected = self.expected_thread_data( @@ -344,11 +353,7 @@ def test_basic_in_blackout_period_with_user_access(self, mock_emit): ) with self.assert_signal_sent( - api, - "thread_created", - sender=None, - user=self.user, - exclude_args=("post", "notify_all_learners"), + api, "thread_created", sender=None, user=self.user, exclude_args=("post", "notify_all_learners") ): actual = create_thread(self.request, self.minimal_data) expected = self.expected_thread_data( @@ -374,7 +379,6 @@ def test_basic_in_blackout_period_with_user_access(self, mock_emit): "type", "voted", ], - "is_deleted": False, } ) assert actual == expected @@ -426,11 +430,7 @@ def test_title_truncation(self, mock_emit): ) self.register_post_thread_response(cs_thread) with self.assert_signal_sent( - api, - "thread_created", - sender=None, - user=self.user, - exclude_args=("post", "notify_all_learners"), + api, "thread_created", sender=None, user=self.user, exclude_args=("post", "notify_all_learners") ): create_thread(self.request, data) event_name, event_data = mock_emit.call_args[0] @@ -718,10 +718,6 @@ def test_success(self, parent_id, mock_emit): "image_url_medium": "http://testserver/static/default_50.png", "image_url_small": "http://testserver/static/default_30.png", }, - "is_deleted": None, - "deleted_at": None, - "deleted_by": None, - "deleted_by_label": None, } assert actual == expected @@ -830,10 +826,6 @@ def test_success_in_black_out_with_user_access(self, parent_id, mock_emit): "image_url_medium": "http://testserver/static/default_50.png", "image_url_small": "http://testserver/static/default_30.png", }, - "is_deleted": False, - "deleted_at": None, - "deleted_by": None, - "deleted_by_label": None, } assert actual == expected @@ -922,9 +914,7 @@ def test_endorsed(self, role_name, is_thread_author, thread_type): ) try: create_comment(self.request, data) - last_commemt_params = self.get_mock_func_calls("create_parent_comment")[-1][ - 1 - ] + last_commemt_params = self.get_mock_func_calls("create_parent_comment")[-1][1] assert last_commemt_params["endorsed"] assert not expected_error except ValidationError: @@ -1838,10 +1828,6 @@ def test_basic(self, parent_id): "image_url_medium": "http://testserver/static/default_50.png", "image_url_small": "http://testserver/static/default_30.png", }, - "is_deleted": None, - "deleted_at": None, - "deleted_by": None, - "deleted_by_label": None, } assert actual == expected params = { @@ -1902,7 +1888,7 @@ def test_abuse_flagged(self, old_flagged, new_flagged, mock_emit): else "edx.forum.response.unreported" ) expected_event_data = { - "discussion": {"id": "test_thread"}, + "discussion": {'id': 'test_thread'}, "body": "Original body", "id": "test_comment", "content_type": "Response", @@ -1965,7 +1951,7 @@ def test_comment_un_abuse_flag_for_moderator_role( "body": "Original body", "id": "test_comment", "content_type": "Response", - "discussion": {"id": "test_thread"}, + "discussion": {'id': 'test_thread'}, "commentable_id": "dummy", "truncated": False, "url": "", @@ -2384,7 +2370,6 @@ def test_basic(self, mock_emit): params = { "thread_id": self.thread_id, "course_id": str(self.course.id), - "deleted_by": str(self.user.id), } self.check_mock_called_with("delete_thread", -1, **params) @@ -2572,7 +2557,6 @@ def test_basic(self, mock_emit): params = { "comment_id": self.comment_id, "course_id": str(self.course.id), - "deleted_by": str(self.user.id), } self.check_mock_called_with("delete_comment", -1, **params) @@ -2937,7 +2921,6 @@ def test_get_threads_by_topic_id(self): "page": 1, "per_page": 1, "commentable_ids": ["topic_x", "topic_meow"], - "show_deleted": False, } self.check_mock_called_with( "get_user_threads", @@ -2953,7 +2936,6 @@ def test_basic_query_params(self): "sort_key": "activity", "page": 6, "per_page": 14, - "show_deleted": False, } self.check_mock_called_with( "get_user_threads", @@ -3094,10 +3076,10 @@ def test_request_group(self, role_name, course_is_cohorted): self.get_thread_list([], course=cohort_course) thread_func_params = self.get_mock_func_calls("get_user_threads")[-1][1] actual_has_group = "group_id" in thread_func_params - expected_has_group = course_is_cohorted and role_name in ( - FORUM_ROLE_STUDENT, - FORUM_ROLE_COMMUNITY_TA, - FORUM_ROLE_GROUP_MODERATOR, + expected_has_group = ( + course_is_cohorted and role_name in ( + FORUM_ROLE_STUDENT, FORUM_ROLE_COMMUNITY_TA, FORUM_ROLE_GROUP_MODERATOR + ) ) assert actual_has_group == expected_has_group @@ -3162,7 +3144,6 @@ def test_text_search(self, text_search_rewrite): "page": 1, "per_page": 10, "text": "test search string", - "show_deleted": False, } self.check_mock_called_with( "search_threads", @@ -3189,7 +3170,6 @@ def test_filter_threads_by_author(self): "page": 1, "per_page": 10, "author_id": str(self.user.id), - "show_deleted": False, } self.check_mock_called_with( "get_user_threads", @@ -3236,7 +3216,6 @@ def test_thread_type(self, thread_type): "page": 1, "per_page": 10, "thread_type": thread_type, - "show_deleted": False, } if thread_type is None: @@ -3274,7 +3253,6 @@ def test_flagged(self, flagged_boolean): "page": 1, "per_page": 10, "flagged": flagged_boolean, - "show_deleted": False, } if flagged_boolean is None: @@ -3315,7 +3293,6 @@ def test_flagged_count(self, role): "count_flagged": True, "page": 1, "per_page": 10, - "show_deleted": False, } self.check_mock_called_with( @@ -3364,7 +3341,6 @@ def test_following(self): "sort_key": "activity", "page": 1, "per_page": 11, - "show_deleted": False, } self.check_mock_called_with("get_user_subscriptions", -1, **params) @@ -3392,7 +3368,6 @@ def test_view_query(self, query): "page": 1, "per_page": 11, query: True, - "show_deleted": False, } self.check_mock_called_with( "get_user_threads", @@ -3434,7 +3409,6 @@ def test_order_by_query(self, http_query, cc_query): "sort_key": cc_query, "page": 1, "per_page": 11, - "show_deleted": False, } self.check_mock_called_with( "get_user_threads", @@ -3467,7 +3441,6 @@ def test_order_direction(self): "sort_key": "activity", "page": 1, "per_page": 11, - "show_deleted": False, } self.check_mock_called_with( "get_user_threads", @@ -3796,10 +3769,6 @@ def get_source_and_expected_comments(self): "image_url_medium": "http://testserver/static/default_50.png", "image_url_small": "http://testserver/static/default_30.png", }, - "is_deleted": None, - "deleted_at": None, - "deleted_by": None, - "deleted_by_label": None, }, { "id": "test_comment_2", @@ -3835,10 +3804,6 @@ def get_source_and_expected_comments(self): "image_url_medium": "http://testserver/static/default_50.png", "image_url_small": "http://testserver/static/default_30.png", }, - "is_deleted": None, - "deleted_at": None, - "deleted_by": None, - "deleted_by_label": None, }, ] return source_comments, expected_comments diff --git a/lms/djangoapps/discussion/rest_api/tests/test_forms.py b/lms/djangoapps/discussion/rest_api/tests/test_forms.py index 33359337933b..3be65964b6b9 100644 --- a/lms/djangoapps/discussion/rest_api/tests/test_forms.py +++ b/lms/djangoapps/discussion/rest_api/tests/test_forms.py @@ -2,6 +2,7 @@ Tests for Discussion API forms """ + import itertools from unittest import TestCase from urllib.parse import urlencode @@ -11,9 +12,9 @@ from opaque_keys.edx.locator import CourseLocator from lms.djangoapps.discussion.rest_api.forms import ( + UserCommentListGetForm, CommentListGetForm, ThreadListGetForm, - UserCommentListGetForm, ) from openedx.core.djangoapps.util.test_forms import FormTestMixin @@ -35,9 +36,7 @@ def test_missing_page_size(self): def test_zero_page_size(self): self.form_data["page_size"] = "0" - self.assert_error( - "page_size", "Ensure this value is greater than or equal to 1." - ) + self.assert_error("page_size", "Ensure this value is greater than or equal to 1.") def test_excessive_page_size(self): self.form_data["page_size"] = "101" @@ -47,7 +46,6 @@ def test_excessive_page_size(self): @ddt.ddt class ThreadListGetFormTest(FormTestMixin, PaginationTestMixin, TestCase): """Tests for ThreadListGetForm""" - FORM_CLASS = ThreadListGetForm def setUp(self): @@ -60,41 +58,37 @@ def setUp(self): "page_size": "13", } ), - mutable=True, + mutable=True ) def test_basic(self): form = self.get_form(expected_valid=True) assert form.cleaned_data == { - "course_id": CourseLocator.from_string("Foo/Bar/Baz"), - "page": 2, - "page_size": 13, - "count_flagged": None, - "topic_id": set(), - "text_search": "", - "following": None, - "author": "", - "thread_type": "", - "flagged": None, - "show_deleted": None, - "view": "", - "order_by": "last_activity_at", - "order_direction": "desc", - "requested_fields": set(), + 'course_id': CourseLocator.from_string('Foo/Bar/Baz'), + 'page': 2, + 'page_size': 13, + 'count_flagged': None, + 'topic_id': set(), + 'text_search': '', + 'following': None, + 'author': '', + 'thread_type': '', + 'flagged': None, + 'view': '', + 'order_by': 'last_activity_at', + 'order_direction': 'desc', + 'requested_fields': set() } def test_topic_id(self): self.form_data.setlist("topic_id", ["example topic_id", "example 2nd topic_id"]) form = self.get_form(expected_valid=True) - assert form.cleaned_data["topic_id"] == { - "example topic_id", - "example 2nd topic_id", - } + assert form.cleaned_data['topic_id'] == {'example topic_id', 'example 2nd topic_id'} def test_text_search(self): self.form_data["text_search"] = "test search string" form = self.get_form(expected_valid=True) - assert form.cleaned_data["text_search"] == "test search string" + assert form.cleaned_data['text_search'] == 'test search string' def test_missing_course_id(self): self.form_data.pop("course_id") @@ -115,10 +109,7 @@ def test_thread_type(self, value): def test_thread_type_invalid(self): self.form_data["thread_type"] = "invalid-option" - self.assert_error( - "thread_type", - "Select a valid choice. invalid-option is not one of the available choices.", - ) + self.assert_error("thread_type", "Select a valid choice. invalid-option is not one of the available choices.") @ddt.data("True", "true", 1, True) def test_flagged_true(self, value): @@ -142,9 +133,7 @@ def test_following_true(self, value): @ddt.data("False", "false", 0, False) def test_following_false(self, value): self.form_data["following"] = value - self.assert_error( - "following", "The value of the 'following' parameter must be true." - ) + self.assert_error("following", "The value of the 'following' parameter must be true.") def test_invalid_following(self): self.form_data["following"] = "invalid-boolean" @@ -155,28 +144,25 @@ def test_mutually_exclusive(self, params): self.form_data.update({param: "True" for param in params}) self.assert_error( "__all__", - "The following query parameters are mutually exclusive: topic_id, text_search, following", + "The following query parameters are mutually exclusive: topic_id, text_search, following" ) def test_invalid_view_choice(self): self.form_data["view"] = "not_a_valid_choice" - self.assert_error( - "view", - "Select a valid choice. not_a_valid_choice is not one of the available choices.", - ) + self.assert_error("view", "Select a valid choice. not_a_valid_choice is not one of the available choices.") def test_invalid_sort_by_choice(self): self.form_data["order_by"] = "not_a_valid_choice" self.assert_error( "order_by", - "Select a valid choice. not_a_valid_choice is not one of the available choices.", + "Select a valid choice. not_a_valid_choice is not one of the available choices." ) def test_invalid_sort_direction_choice(self): self.form_data["order_direction"] = "not_a_valid_choice" self.assert_error( "order_direction", - "Select a valid choice. not_a_valid_choice is not one of the available choices.", + "Select a valid choice. not_a_valid_choice is not one of the available choices." ) @ddt.data( @@ -195,13 +181,12 @@ def test_valid_choice_fields(self, field, value): def test_requested_fields(self): self.form_data["requested_fields"] = "profile_image" form = self.get_form(expected_valid=True) - assert form.cleaned_data["requested_fields"] == {"profile_image"} + assert form.cleaned_data['requested_fields'] == {'profile_image'} @ddt.ddt class CommentListGetFormTest(FormTestMixin, PaginationTestMixin, TestCase): """Tests for CommentListGetForm""" - FORM_CLASS = CommentListGetForm def setUp(self): @@ -217,14 +202,13 @@ def setUp(self): def test_basic(self): form = self.get_form(expected_valid=True) assert form.cleaned_data == { - "thread_id": "deadbeef", - "endorsed": False, - "page": 2, - "page_size": 13, - "flagged": False, - "requested_fields": set(), - "merge_question_type_responses": False, - "show_deleted": None, + 'thread_id': 'deadbeef', + 'endorsed': False, + 'page': 2, + 'page_size': 13, + 'flagged': False, + 'requested_fields': set(), + 'merge_question_type_responses': False } def test_missing_thread_id(self): @@ -252,13 +236,12 @@ def test_invalid_endorsed(self): def test_requested_fields(self): self.form_data["requested_fields"] = {"profile_image"} form = self.get_form(expected_valid=True) - assert form.cleaned_data["requested_fields"] == {"profile_image"} + assert form.cleaned_data['requested_fields'] == {'profile_image'} @ddt.ddt class UserCommentListGetFormTest(FormTestMixin, PaginationTestMixin, TestCase): """Tests for UserCommentListGetForm""" - FORM_CLASS = UserCommentListGetForm def setUp(self): @@ -273,11 +256,11 @@ def setUp(self): def test_basic(self): form = self.get_form(expected_valid=True) assert form.cleaned_data == { - "course_id": CourseLocator.from_string("a/b/c"), - "flagged": False, - "page": 2, - "page_size": 13, - "requested_fields": set(), + 'course_id': CourseLocator.from_string('a/b/c'), + 'flagged': False, + 'page': 2, + 'page_size': 13, + 'requested_fields': set() } def test_missing_flagged(self): @@ -297,7 +280,7 @@ def test_flagged_true(self, value): def test_requested_fields(self): self.form_data["requested_fields"] = {"profile_image"} form = self.get_form(expected_valid=True) - assert form.cleaned_data["requested_fields"] == {"profile_image"} + assert form.cleaned_data['requested_fields'] == {'profile_image'} def test_missing_course_id(self): self.form_data.pop("course_id") diff --git a/lms/djangoapps/discussion/rest_api/tests/test_serializers.py b/lms/djangoapps/discussion/rest_api/tests/test_serializers.py index 10f9b7a64248..a1443252a1ce 100644 --- a/lms/djangoapps/discussion/rest_api/tests/test_serializers.py +++ b/lms/djangoapps/discussion/rest_api/tests/test_serializers.py @@ -9,17 +9,19 @@ import httpretty from django.test.client import RequestFactory from django.test.utils import override_settings +from xmodule.modulestore import ModuleStoreEnum +from xmodule.modulestore.django import modulestore +from xmodule.modulestore.tests.django_utils import SharedModuleStoreTestCase +from xmodule.modulestore.tests.factories import CourseFactory from common.djangoapps.student.tests.factories import UserFactory from common.djangoapps.util.testing import UrlResetMixin -from lms.djangoapps.discussion.django_comment_client.tests.utils import ( - ForumsEnableMixin, -) +from lms.djangoapps.discussion.django_comment_client.tests.utils import ForumsEnableMixin from lms.djangoapps.discussion.rest_api.serializers import ( CommentSerializer, ThreadSerializer, filter_spam_urls_from_html, - get_context, + get_context ) from lms.djangoapps.discussion.rest_api.tests.utils import ( CommentsServiceMockMixin, @@ -37,10 +39,6 @@ FORUM_ROLE_STUDENT, Role, ) -from xmodule.modulestore import ModuleStoreEnum -from xmodule.modulestore.django import modulestore -from xmodule.modulestore.tests.django_utils import SharedModuleStoreTestCase -from xmodule.modulestore.tests.factories import CourseFactory @ddt.ddt @@ -48,18 +46,13 @@ class SerializerTestMixin(ForumsEnableMixin, CommentsServiceMockMixin, UrlResetM """ Test Mixin for Serializer tests """ - @classmethod - @mock.patch.dict( - "django.conf.settings.FEATURES", {"ENABLE_DISCUSSION_SERVICE": True} - ) + @mock.patch.dict("django.conf.settings.FEATURES", {"ENABLE_DISCUSSION_SERVICE": True}) def setUpClass(cls): super().setUpClass() cls.course = CourseFactory.create() - @mock.patch.dict( - "django.conf.settings.FEATURES", {"ENABLE_DISCUSSION_SERVICE": True} - ) + @mock.patch.dict("django.conf.settings.FEATURES", {"ENABLE_DISCUSSION_SERVICE": True}) def setUp(self): super().setUp() httpretty.reset() @@ -67,8 +60,8 @@ def setUp(self): self.addCleanup(httpretty.reset) self.addCleanup(httpretty.disable) patcher = mock.patch( - "openedx.core.djangoapps.discussions.config.waffle.ENABLE_FORUM_V2.is_enabled", - return_value=False, + 'openedx.core.djangoapps.discussions.config.waffle.ENABLE_FORUM_V2.is_enabled', + return_value=False ) patcher.start() self.addCleanup(patcher.stop) @@ -96,9 +89,7 @@ def create_role(self, role_name, users, course=None): (FORUM_ROLE_STUDENT, False, True, True), ) @ddt.unpack - def test_anonymity( - self, role_name, anonymous, anonymous_to_peers, expected_serialized_anonymous - ): + def test_anonymity(self, role_name, anonymous, anonymous_to_peers, expected_serialized_anonymous): """ Test that content is properly made anonymous. @@ -116,9 +107,7 @@ def test_anonymity( """ self.create_role(role_name, [self.user]) serialized = self.serialize( - self.make_cs_content( - {"anonymous": anonymous, "anonymous_to_peers": anonymous_to_peers} - ) + self.make_cs_content({"anonymous": anonymous, "anonymous_to_peers": anonymous_to_peers}) ) actual_serialized_anonymous = serialized["author"] is None assert actual_serialized_anonymous == expected_serialized_anonymous @@ -149,19 +138,17 @@ def test_author_labels(self, role_name, anonymous, expected_label): """ self.create_role(role_name, [self.author]) serialized = self.serialize(self.make_cs_content({"anonymous": anonymous})) - assert serialized["author_label"] == expected_label + assert serialized['author_label'] == expected_label def test_abuse_flagged(self): - serialized = self.serialize( - self.make_cs_content({"abuse_flaggers": [str(self.user.id)]}) - ) - assert serialized["abuse_flagged"] is True + serialized = self.serialize(self.make_cs_content({"abuse_flaggers": [str(self.user.id)]})) + assert serialized['abuse_flagged'] is True def test_voted(self): thread_id = "test_thread" self.register_get_user_response(self.user, upvoted_ids=[thread_id]) serialized = self.serialize(self.make_cs_content({"id": thread_id})) - assert serialized["voted"] is True + assert serialized['voted'] is True @ddt.ddt @@ -188,61 +175,47 @@ def serialize(self, thread): Create a serializer with an appropriate context and use it to serialize the given thread, returning the result. """ - return ThreadSerializer( - thread, context=get_context(self.course, self.request) - ).data + return ThreadSerializer(thread, context=get_context(self.course, self.request)).data def test_basic(self): - thread = make_minimal_cs_thread( - { - "id": "test_thread", - "course_id": str(self.course.id), - "commentable_id": "test_topic", - "user_id": str(self.author.id), - "username": self.author.username, - "title": "Test Title", - "body": "Test body", - "pinned": True, - "votes": {"up_count": 4}, - "comments_count": 5, - "unread_comments_count": 3, - } - ) - expected = self.expected_thread_data( - { - "author": self.author.username, - "can_delete": False, - "vote_count": 4, - "comment_count": 6, - "unread_comment_count": 3, - "pinned": True, - "editable_fields": [ - "abuse_flagged", - "copy_link", - "following", - "read", - "voted", - ], - "abuse_flagged_count": None, - "edit_by_label": None, - "closed_by_label": None, - } - ) + thread = make_minimal_cs_thread({ + "id": "test_thread", + "course_id": str(self.course.id), + "commentable_id": "test_topic", + "user_id": str(self.author.id), + "username": self.author.username, + "title": "Test Title", + "body": "Test body", + "pinned": True, + "votes": {"up_count": 4}, + "comments_count": 5, + "unread_comments_count": 3, + }) + expected = self.expected_thread_data({ + "author": self.author.username, + "can_delete": False, + "vote_count": 4, + "comment_count": 6, + "unread_comment_count": 3, + "pinned": True, + "editable_fields": ["abuse_flagged", "copy_link", "following", "read", "voted"], + "abuse_flagged_count": None, + "edit_by_label": None, + "closed_by_label": None, + }) assert self.serialize(thread) == expected thread["thread_type"] = "question" - expected.update( - { - "type": "question", - "comment_list_url": None, - "endorsed_comment_list_url": ( - "http://testserver/api/discussion/v1/comments/?thread_id=test_thread&endorsed=True" - ), - "non_endorsed_comment_list_url": ( - "http://testserver/api/discussion/v1/comments/?thread_id=test_thread&endorsed=False" - ), - } - ) + expected.update({ + "type": "question", + "comment_list_url": None, + "endorsed_comment_list_url": ( + "http://testserver/api/discussion/v1/comments/?thread_id=test_thread&endorsed=True" + ), + "non_endorsed_comment_list_url": ( + "http://testserver/api/discussion/v1/comments/?thread_id=test_thread&endorsed=False" + ), + }) assert self.serialize(thread) == expected def test_pinned_missing(self): @@ -254,34 +227,34 @@ def test_pinned_missing(self): del thread_data["pinned"] self.register_get_thread_response(thread_data) serialized = self.serialize(thread_data) - assert serialized["pinned"] is False + assert serialized['pinned'] is False def test_group(self): self.course.cohort_config = {"cohorted": True} modulestore().update_item(self.course, ModuleStoreEnum.UserID.test) cohort = CohortFactory.create(course_id=self.course.id) serialized = self.serialize(self.make_cs_content({"group_id": cohort.id})) - assert serialized["group_id"] == cohort.id - assert serialized["group_name"] == cohort.name + assert serialized['group_id'] == cohort.id + assert serialized['group_name'] == cohort.name def test_following(self): thread_id = "test_thread" self.register_get_user_response(self.user, subscribed_thread_ids=[thread_id]) serialized = self.serialize(self.make_cs_content({"id": thread_id})) - assert serialized["following"] is True + assert serialized['following'] is True def test_response_count(self): thread_data = self.make_cs_content({"resp_total": 2}) self.register_get_thread_response(thread_data) serialized = self.serialize(thread_data) - assert serialized["response_count"] == 2 + assert serialized['response_count'] == 2 def test_response_count_missing(self): thread_data = self.make_cs_content({}) del thread_data["resp_total"] self.register_get_thread_response(thread_data) serialized = self.serialize(thread_data) - assert "response_count" not in serialized + assert 'response_count' not in serialized @ddt.data( (FORUM_ROLE_MODERATOR, True), @@ -299,62 +272,43 @@ def test_closed_by_label_field(self, role, visible): self.create_role(FORUM_ROLE_MODERATOR, [moderator]) self.create_role(request_role, [self.user]) - thread = make_minimal_cs_thread( - { - "id": "test_thread", - "course_id": str(self.course.id), - "commentable_id": "test_topic", - "user_id": str(author.id), - "username": author.username, - "title": "Test Title", - "body": "Test body", - "pinned": True, - "votes": {"up_count": 4}, - "comments_count": 5, - "unread_comments_count": 3, - "closed_by": moderator, - } - ) + thread = make_minimal_cs_thread({ + "id": "test_thread", + "course_id": str(self.course.id), + "commentable_id": "test_topic", + "user_id": str(author.id), + "username": author.username, + "title": "Test Title", + "body": "Test body", + "pinned": True, + "votes": {"up_count": 4}, + "comments_count": 5, + "unread_comments_count": 3, + "closed_by": moderator + }) closed_by_label = "Moderator" if visible else None closed_by = moderator if visible else None can_delete = role != FORUM_ROLE_STUDENT editable_fields = ["abuse_flagged", "copy_link", "following", "read", "voted"] if role == "author": editable_fields.remove("voted") - editable_fields.extend( - ["anonymous", "raw_body", "title", "topic_id", "type"] - ) + editable_fields.extend(['anonymous', 'raw_body', 'title', 'topic_id', 'type']) elif role == FORUM_ROLE_MODERATOR: - editable_fields.extend( - [ - "close_reason_code", - "closed", - "edit_reason_code", - "pinned", - "raw_body", - "title", - "topic_id", - "type", - ] - ) - # is_deleted is visible (False) for privileged users, hidden (None) for others - is_deleted = False if role == FORUM_ROLE_MODERATOR else None - expected = self.expected_thread_data( - { - "author": author.username, - "can_delete": can_delete, - "vote_count": 4, - "comment_count": 6, - "unread_comment_count": 3, - "pinned": True, - "editable_fields": sorted(editable_fields), - "abuse_flagged_count": None, - "edit_by_label": None, - "closed_by_label": closed_by_label, - "closed_by": closed_by, - "is_deleted": is_deleted, - } - ) + editable_fields.extend(['close_reason_code', 'closed', 'edit_reason_code', 'pinned', + 'raw_body', 'title', 'topic_id', 'type']) + expected = self.expected_thread_data({ + "author": author.username, + "can_delete": can_delete, + "vote_count": 4, + "comment_count": 6, + "unread_comment_count": 3, + "pinned": True, + "editable_fields": sorted(editable_fields), + "abuse_flagged_count": None, + "edit_by_label": None, + "closed_by_label": closed_by_label, + "closed_by": closed_by, + }) assert self.serialize(thread) == expected @ddt.data( @@ -373,69 +327,48 @@ def test_edit_by_label_field(self, role, visible): self.create_role(FORUM_ROLE_MODERATOR, [moderator]) self.create_role(request_role, [self.user]) - thread = make_minimal_cs_thread( - { - "id": "test_thread", - "course_id": str(self.course.id), - "commentable_id": "test_topic", - "user_id": str(author.id), - "username": author.username, - "title": "Test Title", - "body": "Test body", - "pinned": True, - "votes": {"up_count": 4}, - "edit_history": [{"editor_username": moderator}], - "comments_count": 5, - "unread_comments_count": 3, - "closed_by": None, - } - ) + thread = make_minimal_cs_thread({ + "id": "test_thread", + "course_id": str(self.course.id), + "commentable_id": "test_topic", + "user_id": str(author.id), + "username": author.username, + "title": "Test Title", + "body": "Test body", + "pinned": True, + "votes": {"up_count": 4}, + "edit_history": [{"editor_username": moderator}], + "comments_count": 5, + "unread_comments_count": 3, + "closed_by": None + }) edit_by_label = "Moderator" if visible else None can_delete = role != FORUM_ROLE_STUDENT - last_edit = ( - None if role == FORUM_ROLE_STUDENT else {"editor_username": moderator} - ) + last_edit = None if role == FORUM_ROLE_STUDENT else {"editor_username": moderator} editable_fields = ["abuse_flagged", "copy_link", "following", "read", "voted"] if role == "author": editable_fields.remove("voted") - editable_fields.extend( - ["anonymous", "raw_body", "title", "topic_id", "type"] - ) + editable_fields.extend(['anonymous', 'raw_body', 'title', 'topic_id', 'type']) elif role == FORUM_ROLE_MODERATOR: - editable_fields.extend( - [ - "close_reason_code", - "closed", - "edit_reason_code", - "pinned", - "raw_body", - "title", - "topic_id", - "type", - ] - ) + editable_fields.extend(['close_reason_code', 'closed', 'edit_reason_code', 'pinned', + 'raw_body', 'title', 'topic_id', 'type']) - # is_deleted is visible (False) for privileged users, hidden (None) for others - is_deleted = False if role == FORUM_ROLE_MODERATOR else None - expected = self.expected_thread_data( - { - "author": author.username, - "can_delete": can_delete, - "vote_count": 4, - "comment_count": 6, - "unread_comment_count": 3, - "pinned": True, - "editable_fields": sorted(editable_fields), - "abuse_flagged_count": None, - "last_edit": last_edit, - "edit_by_label": edit_by_label, - "closed_by_label": None, - "closed_by": None, - "is_deleted": is_deleted, - } - ) + expected = self.expected_thread_data({ + "author": author.username, + "can_delete": can_delete, + "vote_count": 4, + "comment_count": 6, + "unread_comment_count": 3, + "pinned": True, + "editable_fields": sorted(editable_fields), + "abuse_flagged_count": None, + "last_edit": last_edit, + "edit_by_label": edit_by_label, + "closed_by_label": None, + "closed_by": None, + }) assert self.serialize(thread) == expected def test_get_preview_body(self): @@ -451,10 +384,7 @@ def test_get_preview_body(self): {"body": "

    This is a test thread body with some text.

    "} ) serialized = self.serialize(thread_data) - assert ( - serialized["preview_body"] - == "This is a test thread body with some text." - ) + assert serialized['preview_body'] == "This is a test thread body with some text." @ddt.ddt @@ -472,12 +402,12 @@ def make_cs_content(self, overrides=None, with_endorsement=False): """ merged_overrides = { "user_id": str(self.author.id), - "username": self.author.username, + "username": self.author.username } if with_endorsement: merged_overrides["endorsement"] = { "user_id": str(self.endorser.id), - "time": self.endorsed_at, + "time": self.endorsed_at } merged_overrides.update(overrides or {}) return make_minimal_cs_comment(merged_overrides) @@ -487,9 +417,7 @@ def serialize(self, comment, thread_data=None): Create a serializer with an appropriate context and use it to serialize the given comment, returning the result. """ - context = get_context( - self.course, self.request, make_minimal_cs_thread(thread_data) - ) + context = get_context(self.course, self.request, make_minimal_cs_thread(thread_data)) return CommentSerializer(comment, context=context).data def test_basic(self): @@ -544,10 +472,6 @@ def test_basic(self): "image_url_medium": "http://testserver/static/default_50.png", "image_url_small": "http://testserver/static/default_30.png", }, - "is_deleted": None, - "deleted_at": None, - "deleted_by": None, - "deleted_by_label": None, } assert self.serialize(comment) == expected @@ -560,7 +484,7 @@ def test_basic(self): FORUM_ROLE_COMMUNITY_TA, FORUM_ROLE_STUDENT, ], - [True, False], + [True, False] ) ) @ddt.unpack @@ -577,12 +501,10 @@ def test_endorsed_by(self, endorser_role_name, thread_anonymous): self.create_role(endorser_role_name, [self.endorser]) serialized = self.serialize( self.make_cs_content(with_endorsement=True), - thread_data={"anonymous": thread_anonymous}, + thread_data={"anonymous": thread_anonymous} ) actual_endorser_anonymous = serialized["endorsed_by"] is None - expected_endorser_anonymous = ( - endorser_role_name == FORUM_ROLE_STUDENT and thread_anonymous - ) + expected_endorser_anonymous = endorser_role_name == FORUM_ROLE_STUDENT and thread_anonymous assert actual_endorser_anonymous == expected_endorser_anonymous @ddt.data( @@ -605,69 +527,56 @@ def test_endorsed_by_labels(self, role_name, expected_label): """ self.create_role(role_name, [self.endorser]) serialized = self.serialize(self.make_cs_content(with_endorsement=True)) - assert serialized["endorsed_by_label"] == expected_label + assert serialized['endorsed_by_label'] == expected_label def test_endorsed_at(self): serialized = self.serialize(self.make_cs_content(with_endorsement=True)) - assert serialized["endorsed_at"] == self.endorsed_at + assert serialized['endorsed_at'] == self.endorsed_at def test_children(self): - comment = self.make_cs_content( - { - "id": "test_root", - "children": [ - self.make_cs_content( - { - "id": "test_child_1", - "parent_id": "test_root", - } - ), - self.make_cs_content( - { - "id": "test_child_2", - "parent_id": "test_root", - "children": [ - self.make_cs_content( - { - "id": "test_grandchild", - "parent_id": "test_child_2", - } - ) - ], - } - ), - ], - } - ) + comment = self.make_cs_content({ + "id": "test_root", + "children": [ + self.make_cs_content({ + "id": "test_child_1", + "parent_id": "test_root", + }), + self.make_cs_content({ + "id": "test_child_2", + "parent_id": "test_root", + "children": [ + self.make_cs_content({ + "id": "test_grandchild", + "parent_id": "test_child_2" + }) + ], + }), + ], + }) serialized = self.serialize(comment) - assert serialized["children"][0]["id"] == "test_child_1" - assert serialized["children"][0]["parent_id"] == "test_root" - assert serialized["children"][1]["id"] == "test_child_2" - assert serialized["children"][1]["parent_id"] == "test_root" - assert serialized["children"][1]["children"][0]["id"] == "test_grandchild" - assert serialized["children"][1]["children"][0]["parent_id"] == "test_child_2" + assert serialized['children'][0]['id'] == 'test_child_1' + assert serialized['children'][0]['parent_id'] == 'test_root' + assert serialized['children'][1]['id'] == 'test_child_2' + assert serialized['children'][1]['parent_id'] == 'test_root' + assert serialized['children'][1]['children'][0]['id'] == 'test_grandchild' + assert serialized['children'][1]['children'][0]['parent_id'] == 'test_child_2' @ddt.ddt class ThreadSerializerDeserializationTest( - ForumsEnableMixin, - CommentsServiceMockMixin, - UrlResetMixin, - SharedModuleStoreTestCase, + ForumsEnableMixin, + CommentsServiceMockMixin, + UrlResetMixin, + SharedModuleStoreTestCase ): """Tests for ThreadSerializer deserialization.""" - @classmethod - @mock.patch.dict( - "django.conf.settings.FEATURES", {"ENABLE_DISCUSSION_SERVICE": True} - ) + @mock.patch.dict("django.conf.settings.FEATURES", {"ENABLE_DISCUSSION_SERVICE": True}) def setUpClass(cls): super().setUpClass() cls.course = CourseFactory.create() - @mock.patch.dict( - "django.conf.settings.FEATURES", {"ENABLE_DISCUSSION_SERVICE": True} - ) + @mock.patch.dict("django.conf.settings.FEATURES", {"ENABLE_DISCUSSION_SERVICE": True}) def setUp(self): super().setUp() httpretty.reset() @@ -675,8 +584,8 @@ def setUp(self): self.addCleanup(httpretty.reset) self.addCleanup(httpretty.disable) patcher = mock.patch( - "openedx.core.djangoapps.discussions.config.waffle.ENABLE_FORUM_V2.is_enabled", - return_value=False, + 'openedx.core.djangoapps.discussions.config.waffle.ENABLE_FORUM_V2.is_enabled', + return_value=False ) patcher.start() self.addCleanup(patcher.stop) @@ -691,22 +600,18 @@ def setUp(self): "title": "Test Title", "raw_body": "Test body", } - self.existing_thread = Thread( - **make_minimal_cs_thread( - { - "id": "existing_thread", - "course_id": str(self.course.id), - "commentable_id": "original_topic", - "thread_type": "discussion", - "title": "Original Title", - "body": "Original body", - "user_id": str(self.user.id), - "username": self.user.username, - "read": "False", - "endorsed": "False", - } - ) - ) + self.existing_thread = Thread(**make_minimal_cs_thread({ + "id": "existing_thread", + "course_id": str(self.course.id), + "commentable_id": "original_topic", + "thread_type": "discussion", + "title": "Original Title", + "body": "Original body", + "user_id": str(self.user.id), + "username": self.user.username, + "read": "False", + "endorsed": "False" + })) def save_and_reserialize(self, data, instance=None): """ @@ -718,7 +623,7 @@ def save_and_reserialize(self, data, instance=None): instance, data=data, partial=(instance is not None), - context=get_context(self.course, self.request), + context=get_context(self.course, self.request) ) assert serializer.is_valid() serializer.save() @@ -730,36 +635,33 @@ def test_create_missing_field(self): data.pop(field) serializer = ThreadSerializer(data=data) assert not serializer.is_valid() - assert serializer.errors == {field: ["This field is required."]} + assert serializer.errors == {field: ['This field is required.']} @ddt.data("", " ") def test_create_empty_string(self, value): data = self.minimal_data.copy() data.update({field: value for field in ["topic_id", "title", "raw_body"]}) - serializer = ThreadSerializer( - data=data, context=get_context(self.course, self.request) - ) + serializer = ThreadSerializer(data=data, context=get_context(self.course, self.request)) assert not serializer.is_valid() assert serializer.errors == { - field: ["This field may not be blank."] - for field in ["topic_id", "title", "raw_body"] + field: ['This field may not be blank.'] for field in ['topic_id', 'title', 'raw_body'] } def test_update_empty(self): self.register_put_thread_response(self.existing_thread.attributes) self.save_and_reserialize({}, self.existing_thread) assert parsed_body(httpretty.last_request()) == { - "course_id": [str(self.course.id)], - "commentable_id": ["original_topic"], - "thread_type": ["discussion"], - "title": ["Original Title"], - "body": ["Original body"], - "anonymous": ["False"], - "anonymous_to_peers": ["False"], - "closed": ["False"], - "pinned": ["False"], - "user_id": [str(self.user.id)], - "read": ["False"], + 'course_id': [str(self.course.id)], + 'commentable_id': ['original_topic'], + 'thread_type': ['discussion'], + 'title': ['Original Title'], + 'body': ['Original body'], + 'anonymous': ['False'], + 'anonymous_to_peers': ['False'], + 'closed': ['False'], + 'pinned': ['False'], + 'user_id': [str(self.user.id)], + 'read': ['False'] } @ddt.data(True, False) @@ -774,18 +676,18 @@ def test_update_all(self, read): } saved = self.save_and_reserialize(data, self.existing_thread) assert parsed_body(httpretty.last_request()) == { - "course_id": [str(self.course.id)], - "commentable_id": ["edited_topic"], - "thread_type": ["question"], - "title": ["Edited Title"], - "body": ["Edited body"], - "anonymous": ["False"], - "anonymous_to_peers": ["False"], - "closed": ["False"], - "pinned": ["False"], - "user_id": [str(self.user.id)], - "read": [str(read)], - "editing_user_id": [str(self.user.id)], + 'course_id': [str(self.course.id)], + 'commentable_id': ['edited_topic'], + 'thread_type': ['question'], + 'title': ['Edited Title'], + 'body': ['Edited body'], + 'anonymous': ['False'], + 'anonymous_to_peers': ['False'], + 'closed': ['False'], + 'pinned': ['False'], + 'user_id': [str(self.user.id)], + 'read': [str(read)], + 'editing_user_id': [str(self.user.id)], } for key in data: assert saved[key] == data[key] @@ -800,7 +702,7 @@ def test_update_anonymous(self): "anonymous": True, } self.save_and_reserialize(data, self.existing_thread) - assert parsed_body(httpretty.last_request())["anonymous"] == ["True"] + assert parsed_body(httpretty.last_request())["anonymous"] == ['True'] def test_update_anonymous_to_peers(self): """ @@ -812,7 +714,7 @@ def test_update_anonymous_to_peers(self): "anonymous_to_peers": True, } self.save_and_reserialize(data, self.existing_thread) - assert parsed_body(httpretty.last_request())["anonymous_to_peers"] == ["True"] + assert parsed_body(httpretty.last_request())["anonymous_to_peers"] == ['True'] @ddt.data("", " ") def test_update_empty_string(self, value): @@ -820,12 +722,11 @@ def test_update_empty_string(self, value): self.existing_thread, data={field: value for field in ["topic_id", "title", "raw_body"]}, partial=True, - context=get_context(self.course, self.request), + context=get_context(self.course, self.request) ) assert not serializer.is_valid() assert serializer.errors == { - field: ["This field may not be blank."] - for field in ["topic_id", "title", "raw_body"] + field: ['This field may not be blank.'] for field in ['topic_id', 'title', 'raw_body'] } def test_update_course_id(self): @@ -833,20 +734,15 @@ def test_update_course_id(self): self.existing_thread, data={"course_id": "some/other/course"}, partial=True, - context=get_context(self.course, self.request), + context=get_context(self.course, self.request) ) assert not serializer.is_valid() - assert serializer.errors == { - "course_id": ["This field is not allowed in an update."] - } + assert serializer.errors == {'course_id': ['This field is not allowed in an update.']} @ddt.ddt -class CommentSerializerDeserializationTest( - ForumsEnableMixin, CommentsServiceMockMixin, SharedModuleStoreTestCase -): +class CommentSerializerDeserializationTest(ForumsEnableMixin, CommentsServiceMockMixin, SharedModuleStoreTestCase): """Tests for ThreadSerializer deserialization.""" - @classmethod def setUpClass(cls): super().setUpClass() @@ -859,8 +755,8 @@ def setUp(self): self.addCleanup(httpretty.reset) self.addCleanup(httpretty.disable) patcher = mock.patch( - "openedx.core.djangoapps.discussions.config.waffle.ENABLE_FORUM_V2.is_enabled", - return_value=False, + 'openedx.core.djangoapps.discussions.config.waffle.ENABLE_FORUM_V2.is_enabled', + return_value=False ) patcher.start() self.addCleanup(patcher.stop) @@ -882,18 +778,14 @@ def setUp(self): "thread_id": "test_thread", "raw_body": "Test body", } - self.existing_comment = Comment( - **make_minimal_cs_comment( - { - "id": "existing_comment", - "thread_id": "dummy", - "body": "Original body", - "user_id": str(self.user.id), - "username": self.user.username, - "course_id": str(self.course.id), - } - ) - ) + self.existing_comment = Comment(**make_minimal_cs_comment({ + "id": "existing_comment", + "thread_id": "dummy", + "body": "Original body", + "user_id": str(self.user.id), + "username": self.user.username, + "course_id": str(self.course.id), + })) def save_and_reserialize(self, data, instance=None): """ @@ -903,10 +795,13 @@ def save_and_reserialize(self, data, instance=None): context = get_context( self.course, self.request, - make_minimal_cs_thread({"course_id": str(self.course.id)}), + make_minimal_cs_thread({"course_id": str(self.course.id)}) ) serializer = CommentSerializer( - instance, data=data, partial=(instance is not None), context=context + instance, + data=data, + partial=(instance is not None), + context=context ) assert serializer.is_valid() serializer.save() @@ -918,23 +813,21 @@ def test_create_missing_field(self): data.pop(field) serializer = CommentSerializer( data=data, - context=get_context( - self.course, self.request, make_minimal_cs_thread() - ), + context=get_context(self.course, self.request, make_minimal_cs_thread()) ) assert not serializer.is_valid() - assert serializer.errors == {field: ["This field is required."]} + assert serializer.errors == {field: ['This field is required.']} def test_update_empty(self): self.register_put_comment_response(self.existing_comment.attributes) self.save_and_reserialize({}, instance=self.existing_comment) assert parsed_body(httpretty.last_request()) == { - "body": ["Original body"], - "course_id": [str(self.course.id)], - "user_id": [str(self.user.id)], - "anonymous": ["False"], - "anonymous_to_peers": ["False"], - "endorsed": ["False"], + 'body': ['Original body'], + 'course_id': [str(self.course.id)], + 'user_id': [str(self.user.id)], + 'anonymous': ['False'], + 'anonymous_to_peers': ['False'], + 'endorsed': ['False'] } def test_update_anonymous(self): @@ -947,7 +840,7 @@ def test_update_anonymous(self): "anonymous": True, } self.save_and_reserialize(data, self.existing_comment) - assert parsed_body(httpretty.last_request())["anonymous"] == ["True"] + assert parsed_body(httpretty.last_request())["anonymous"] == ['True'] def test_update_anonymous_to_peers(self): """ @@ -959,7 +852,7 @@ def test_update_anonymous_to_peers(self): "anonymous_to_peers": True, } self.save_and_reserialize(data, self.existing_comment) - assert parsed_body(httpretty.last_request())["anonymous_to_peers"] == ["True"] + assert parsed_body(httpretty.last_request())["anonymous_to_peers"] == ['True'] @ddt.data("thread_id", "parent_id") def test_update_non_updatable(self, field): @@ -967,26 +860,23 @@ def test_update_non_updatable(self, field): self.existing_comment, data={field: "different_value"}, partial=True, - context=get_context(self.course, self.request), + context=get_context(self.course, self.request) ) assert not serializer.is_valid() - assert serializer.errors == {field: ["This field is not allowed in an update."]} + assert serializer.errors == {field: ['This field is not allowed in an update.']} class FilterSpamTest(SharedModuleStoreTestCase): """ Tests for the filter_spam method """ - - @override_settings(DISCUSSION_SPAM_URLS=["example.com"]) + @override_settings(DISCUSSION_SPAM_URLS=['example.com']) def test_filter(self): self.assertEqual( - filter_spam_urls_from_html( - '' - )[0], - "
    abc
    ", + filter_spam_urls_from_html('')[0], + '
    abc
    ' ) self.assertEqual( - filter_spam_urls_from_html("
    example.com/abc/def
    ")[0], - "
    ", + filter_spam_urls_from_html('
    example.com/abc/def
    ')[0], + '
    ' ) diff --git a/lms/djangoapps/discussion/rest_api/tests/test_views.py b/lms/djangoapps/discussion/rest_api/tests/test_views.py index 8a9405076c2d..e4d46168c46d 100644 --- a/lms/djangoapps/discussion/rest_api/tests/test_views.py +++ b/lms/djangoapps/discussion/rest_api/tests/test_views.py @@ -2,6 +2,7 @@ Tests for Discussion API views """ + import json import random from datetime import datetime @@ -19,22 +20,22 @@ from rest_framework import status from rest_framework.test import APIClient, APITestCase +from lms.djangoapps.discussion.toggles import ENABLE_DISCUSSIONS_MFE +from lms.djangoapps.discussion.rest_api.utils import get_usernames_from_search_string +from xmodule.modulestore import ModuleStoreEnum +from xmodule.modulestore.django import modulestore +from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase, SharedModuleStoreTestCase +from xmodule.modulestore.tests.factories import CourseFactory, BlockFactory, check_mongo_calls + from common.djangoapps.course_modes.models import CourseMode from common.djangoapps.course_modes.tests.factories import CourseModeFactory -from common.djangoapps.student.models import ( - CourseEnrollment, - get_retired_username_by_username, -) -from common.djangoapps.student.roles import ( - CourseInstructorRole, - CourseStaffRole, - GlobalStaff, -) +from common.djangoapps.student.models import get_retired_username_by_username, CourseEnrollment +from common.djangoapps.student.roles import CourseInstructorRole, CourseStaffRole, GlobalStaff from common.djangoapps.student.tests.factories import ( AdminFactory, CourseEnrollmentFactory, SuperuserFactory, - UserFactory, + UserFactory ) from common.djangoapps.util.testing import UrlResetMixin from lms.djangoapps.discussion.django_comment_client.tests.utils import ( @@ -47,50 +48,21 @@ make_minimal_cs_comment, make_minimal_cs_thread, ) -from lms.djangoapps.discussion.rest_api.utils import get_usernames_from_search_string -from lms.djangoapps.discussion.toggles import ENABLE_DISCUSSIONS_MFE from openedx.core.djangoapps.course_groups.tests.helpers import config_course_cohorts -from openedx.core.djangoapps.discussions.config.waffle import ( - ENABLE_NEW_STRUCTURE_DISCUSSIONS, -) -from openedx.core.djangoapps.discussions.models import ( - DiscussionsConfiguration, - DiscussionTopicLink, - Provider, -) -from openedx.core.djangoapps.discussions.tasks import ( - update_discussions_settings_from_course_task, -) +from openedx.core.djangoapps.discussions.config.waffle import ENABLE_NEW_STRUCTURE_DISCUSSIONS +from openedx.core.djangoapps.discussions.models import DiscussionsConfiguration, DiscussionTopicLink, Provider +from openedx.core.djangoapps.discussions.tasks import update_discussions_settings_from_course_task from openedx.core.djangoapps.django_comment_common.models import ( CourseDiscussionSettings, Role, ) from openedx.core.djangoapps.django_comment_common.utils import seed_permissions_roles from openedx.core.djangoapps.oauth_dispatch.jwt import create_jwt_for_user -from openedx.core.djangoapps.oauth_dispatch.tests.factories import ( - AccessTokenFactory, - ApplicationFactory, -) -from openedx.core.djangoapps.user_api.models import ( - RetirementState, - UserRetirementStatus, -) -from xmodule.modulestore import ModuleStoreEnum -from xmodule.modulestore.django import modulestore -from xmodule.modulestore.tests.django_utils import ( - ModuleStoreTestCase, - SharedModuleStoreTestCase, -) -from xmodule.modulestore.tests.factories import ( - BlockFactory, - CourseFactory, - check_mongo_calls, -) +from openedx.core.djangoapps.oauth_dispatch.tests.factories import AccessTokenFactory, ApplicationFactory +from openedx.core.djangoapps.user_api.models import RetirementState, UserRetirementStatus -class DiscussionAPIViewTestMixin( - ForumsEnableMixin, CommentsServiceMockMixin, UrlResetMixin -): +class DiscussionAPIViewTestMixin(ForumsEnableMixin, CommentsServiceMockMixin, UrlResetMixin): """ Mixin for common code in tests of Discussion API views. This includes creation of common structures (e.g. a course, user, and enrollment), logging @@ -100,9 +72,7 @@ class DiscussionAPIViewTestMixin( client_class = APIClient - @mock.patch.dict( - "django.conf.settings.FEATURES", {"ENABLE_DISCUSSION_SERVICE": True} - ) + @mock.patch.dict("django.conf.settings.FEATURES", {"ENABLE_DISCUSSION_SERVICE": True}) def setUp(self): super().setUp() self.maxDiff = None # pylint: disable=invalid-name @@ -111,7 +81,7 @@ def setUp(self): course="y", run="z", start=datetime.now(UTC), - discussion_topics={"Test Topic": {"id": "test_topic"}}, + discussion_topics={"Test Topic": {"id": "test_topic"}} ) self.password = "Password1234" self.user = UserFactory.create(password=self.password) @@ -126,25 +96,23 @@ def assert_response_correct(self, response, expected_status, expected_content): Assert that the response has the given status code and parsed content """ assert response.status_code == expected_status - parsed_content = json.loads(response.content.decode("utf-8")) + parsed_content = json.loads(response.content.decode('utf-8')) assert parsed_content == expected_content def register_thread(self, overrides=None): """ Create cs_thread with minimal fields and register response """ - cs_thread = make_minimal_cs_thread( - { - "id": "test_thread", - "course_id": str(self.course.id), - "commentable_id": "test_topic", - "username": self.user.username, - "user_id": str(self.user.id), - "thread_type": "discussion", - "title": "Test Title", - "body": "Test body", - } - ) + cs_thread = make_minimal_cs_thread({ + "id": "test_thread", + "course_id": str(self.course.id), + "commentable_id": "test_topic", + "username": self.user.username, + "user_id": str(self.user.id), + "thread_type": "discussion", + "title": "Test Title", + "body": "Test body", + }) cs_thread.update(overrides or {}) self.register_get_thread_response(cs_thread) self.register_put_thread_response(cs_thread) @@ -153,16 +121,14 @@ def register_comment(self, overrides=None): """ Create cs_comment with minimal fields and register response """ - cs_comment = make_minimal_cs_comment( - { - "id": "test_comment", - "course_id": str(self.course.id), - "thread_id": "test_thread", - "username": self.user.username, - "user_id": str(self.user.id), - "body": "Original body", - } - ) + cs_comment = make_minimal_cs_comment({ + "id": "test_comment", + "course_id": str(self.course.id), + "thread_id": "test_thread", + "username": self.user.username, + "user_id": str(self.user.id), + "body": "Original body", + }) cs_comment.update(overrides or {}) self.register_get_comment_response(cs_comment) self.register_put_comment_response(cs_comment) @@ -174,7 +140,7 @@ def test_not_authenticated(self): self.assert_response_correct( response, 401, - {"developer_message": "Authentication credentials were not provided."}, + {"developer_message": "Authentication credentials were not provided."} ) def test_inactive(self): @@ -183,16 +149,12 @@ def test_inactive(self): @mock.patch.dict("django.conf.settings.FEATURES", {"ENABLE_DISCUSSION_SERVICE": True}) -class UploadFileViewTest( - ForumsEnableMixin, CommentsServiceMockMixin, UrlResetMixin, ModuleStoreTestCase -): +class UploadFileViewTest(ForumsEnableMixin, CommentsServiceMockMixin, UrlResetMixin, ModuleStoreTestCase): """ Tests for UploadFileView. """ - @mock.patch.dict( - "django.conf.settings.FEATURES", {"ENABLE_DISCUSSION_SERVICE": True} - ) + @mock.patch.dict("django.conf.settings.FEATURES", {"ENABLE_DISCUSSION_SERVICE": True}) def setUp(self): super().setUp() self.valid_file = { @@ -203,13 +165,11 @@ def setUp(self): ), } self.user = UserFactory.create(password=self.TEST_PASSWORD) - self.course = CourseFactory.create( - org="a", course="b", run="c", start=datetime.now(UTC) - ) + self.course = CourseFactory.create(org='a', course='b', run='c', start=datetime.now(UTC)) self.url = reverse("upload_file", kwargs={"course_id": str(self.course.id)}) patcher = mock.patch( - "openedx.core.djangoapps.discussions.config.waffle.ENABLE_FORUM_V2.is_enabled", - return_value=False, + 'openedx.core.djangoapps.discussions.config.waffle.ENABLE_FORUM_V2.is_enabled', + return_value=False ) patcher.start() self.addCleanup(patcher.stop) @@ -297,13 +257,10 @@ def test_file_upload_with_thread_key(self): """ self.user_login() self.enroll_user_in_course() - response = self.client.post( - self.url, - { - **self.valid_file, - "thread_key": "somethread", - }, - ) + response = self.client.post(self.url, { + **self.valid_file, + "thread_key": "somethread", + }) response_data = json.loads(response.content) assert "/somethread/" in response_data["location"] @@ -357,9 +314,7 @@ class CommentViewSetListByUserTest( Common test cases for views retrieving user-published content. """ - @mock.patch.dict( - "django.conf.settings.FEATURES", {"ENABLE_DISCUSSION_SERVICE": True} - ) + @mock.patch.dict("django.conf.settings.FEATURES", {"ENABLE_DISCUSSION_SERVICE": True}) def setUp(self): super().setUp() @@ -368,8 +323,8 @@ def setUp(self): self.addCleanup(httpretty.reset) self.addCleanup(httpretty.disable) patcher = mock.patch( - "openedx.core.djangoapps.discussions.config.waffle.ENABLE_FORUM_V2.is_enabled", - return_value=False, + 'openedx.core.djangoapps.discussions.config.waffle.ENABLE_FORUM_V2.is_enabled', + return_value=False ) patcher.start() self.addCleanup(patcher.stop) @@ -380,9 +335,7 @@ def setUp(self): self.other_user = UserFactory.create(password=self.TEST_PASSWORD) self.register_get_user_response(self.other_user) - self.course = CourseFactory.create( - org="a", course="b", run="c", start=datetime.now(UTC) - ) + self.course = CourseFactory.create(org="a", course="b", run="c", start=datetime.now(UTC)) CourseEnrollmentFactory.create(user=self.user, course_id=self.course.id) self.url = self.build_url(self.user.username, self.course.id) @@ -393,18 +346,16 @@ def register_mock_endpoints(self): """ self.register_get_threads_response( threads=[ - make_minimal_cs_thread( - { - "id": f"test_thread_{index}", - "course_id": str(self.course.id), - "commentable_id": f"test_topic_{index}", - "username": self.user.username, - "user_id": str(self.user.id), - "thread_type": "discussion", - "title": f"Test Title #{index}", - "body": f"Test body #{index}", - } - ) + make_minimal_cs_thread({ + "id": f"test_thread_{index}", + "course_id": str(self.course.id), + "commentable_id": f"test_topic_{index}", + "username": self.user.username, + "user_id": str(self.user.id), + "thread_type": "discussion", + "title": f"Test Title #{index}", + "body": f"Test body #{index}", + }) for index in range(30) ], page=1, @@ -412,18 +363,16 @@ def register_mock_endpoints(self): ) self.register_get_comments_response( comments=[ - make_minimal_cs_comment( - { - "id": f"test_comment_{index}", - "thread_id": "test_thread", - "user_id": str(self.user.id), - "username": self.user.username, - "created_at": "2015-05-11T00:00:00Z", - "updated_at": "2015-05-11T11:11:11Z", - "body": f"Test body #{index}", - "votes": {"up_count": 4}, - } - ) + make_minimal_cs_comment({ + "id": f"test_comment_{index}", + "thread_id": "test_thread", + "user_id": str(self.user.id), + "username": self.user.username, + "created_at": "2015-05-11T00:00:00Z", + "updated_at": "2015-05-11T11:11:11Z", + "body": f"Test body #{index}", + "votes": {"up_count": 4}, + }) for index in range(30) ], page=1, @@ -435,13 +384,11 @@ def build_url(self, username, course_id, **kwargs): Builds an URL to access content from an user on a specific course. """ base = reverse("comment-list") - query = urlencode( - { - "username": username, - "course_id": str(course_id), - **kwargs, - } - ) + query = urlencode({ + "username": username, + "course_id": str(course_id), + **kwargs, + }) return f"{base}?{query}" def assert_successful_response(self, response): @@ -467,9 +414,7 @@ def test_request_by_unauthorized_user(self): they're not either enrolled or staff members. """ self.register_mock_endpoints() - self.client.login( - username=self.other_user.username, password=self.TEST_PASSWORD - ) + self.client.login(username=self.other_user.username, password=self.TEST_PASSWORD) response = self.client.get(self.url) assert response.status_code == status.HTTP_404_NOT_FOUND assert json.loads(response.content)["developer_message"] == "Course not found." @@ -480,9 +425,7 @@ def test_request_by_enrolled_user(self): comments in that course. """ self.register_mock_endpoints() - self.client.login( - username=self.other_user.username, password=self.TEST_PASSWORD - ) + self.client.login(username=self.other_user.username, password=self.TEST_PASSWORD) CourseEnrollmentFactory.create(user=self.other_user, course_id=self.course.id) self.assert_successful_response(self.client.get(self.url)) @@ -491,9 +434,7 @@ def test_request_by_global_staff(self): Staff users are allowed to get any user's comments. """ self.register_mock_endpoints() - self.client.login( - username=self.other_user.username, password=self.TEST_PASSWORD - ) + self.client.login(username=self.other_user.username, password=self.TEST_PASSWORD) GlobalStaff().add_users(self.other_user) self.assert_successful_response(self.client.get(self.url)) @@ -504,9 +445,7 @@ def test_request_by_course_staff(self, role): course. """ self.register_mock_endpoints() - self.client.login( - username=self.other_user.username, password=self.TEST_PASSWORD - ) + self.client.login(username=self.other_user.username, password=self.TEST_PASSWORD) role(course_key=self.course.id).add_users(self.other_user) self.assert_successful_response(self.client.get(self.url)) @@ -515,9 +454,7 @@ def test_request_with_non_existent_user(self): Requests for users that don't exist result in a 404 response. """ self.register_mock_endpoints() - self.client.login( - username=self.other_user.username, password=self.TEST_PASSWORD - ) + self.client.login(username=self.other_user.username, password=self.TEST_PASSWORD) GlobalStaff().add_users(self.other_user) url = self.build_url("non_existent", self.course.id) response = self.client.get(url) @@ -528,9 +465,7 @@ def test_request_with_non_existent_course(self): Requests for courses that don't exist result in a 404 response. """ self.register_mock_endpoints() - self.client.login( - username=self.other_user.username, password=self.TEST_PASSWORD - ) + self.client.login(username=self.other_user.username, password=self.TEST_PASSWORD) GlobalStaff().add_users(self.other_user) url = self.build_url(self.user.username, "course-v1:x+y+z") response = self.client.get(url) @@ -541,18 +476,14 @@ def test_request_with_invalid_course_id(self): Requests with invalid course ID should fail form validation. """ self.register_mock_endpoints() - self.client.login( - username=self.other_user.username, password=self.TEST_PASSWORD - ) + self.client.login(username=self.other_user.username, password=self.TEST_PASSWORD) GlobalStaff().add_users(self.other_user) url = self.build_url(self.user.username, "an invalid course") response = self.client.get(url) assert response.status_code == status.HTTP_400_BAD_REQUEST parsed_response = json.loads(response.content) - assert ( - parsed_response["field_errors"]["course_id"]["developer_message"] - == "'an invalid course' is not a valid course id" - ) + assert parsed_response["field_errors"]["course_id"]["developer_message"] == \ + "'an invalid course' is not a valid course id" def test_request_with_empty_results_page(self): """ @@ -562,9 +493,7 @@ def test_request_with_empty_results_page(self): self.register_get_threads_response(threads=[], page=1, num_pages=1) self.register_get_comments_response(comments=[], page=1, num_pages=1) - self.client.login( - username=self.other_user.username, password=self.TEST_PASSWORD - ) + self.client.login(username=self.other_user.username, password=self.TEST_PASSWORD) GlobalStaff().add_users(self.other_user) url = self.build_url(self.user.username, self.course.id, page=2) response = self.client.get(url) @@ -572,23 +501,17 @@ def test_request_with_empty_results_page(self): @mock.patch.dict("django.conf.settings.FEATURES", {"ENABLE_DISCUSSION_SERVICE": True}) -@override_settings( - DISCUSSION_MODERATION_EDIT_REASON_CODES={"test-edit-reason": "Test Edit Reason"} -) -@override_settings( - DISCUSSION_MODERATION_CLOSE_REASON_CODES={"test-close-reason": "Test Close Reason"} -) +@override_settings(DISCUSSION_MODERATION_EDIT_REASON_CODES={"test-edit-reason": "Test Edit Reason"}) +@override_settings(DISCUSSION_MODERATION_CLOSE_REASON_CODES={"test-close-reason": "Test Close Reason"}) class CourseViewTest(DiscussionAPIViewTestMixin, ModuleStoreTestCase): """Tests for CourseView""" def setUp(self): super().setUp() - self.url = reverse( - "discussion_course", kwargs={"course_id": str(self.course.id)} - ) + self.url = reverse("discussion_course", kwargs={"course_id": str(self.course.id)}) patcher = mock.patch( - "openedx.core.djangoapps.discussions.config.waffle.ENABLE_FORUM_V2.is_enabled", - return_value=False, + 'openedx.core.djangoapps.discussions.config.waffle.ENABLE_FORUM_V2.is_enabled', + return_value=False ) patcher.start() self.addCleanup(patcher.stop) @@ -598,7 +521,9 @@ def test_404(self): reverse("course_topics", kwargs={"course_id": "non/existent/course"}) ) self.assert_response_correct( - response, 404, {"developer_message": "Course not found."} + response, + 404, + {"developer_message": "Course not found."} ) def test_basic(self): @@ -622,27 +547,23 @@ def test_basic(self): "allow_anonymous_to_peers": False, "has_bulk_delete_privileges": False, "has_moderation_privileges": False, - "is_course_admin": False, - "is_course_staff": False, + 'is_course_admin': False, + 'is_course_staff': False, "is_group_ta": False, - "is_user_admin": False, + 'is_user_admin': False, "user_roles": ["Student"], - "edit_reasons": [ - {"code": "test-edit-reason", "label": "Test Edit Reason"} - ], - "post_close_reasons": [ - {"code": "test-close-reason", "label": "Test Close Reason"} - ], - "show_discussions": True, - "is_notify_all_learners_enabled": False, - "captcha_settings": { - "enabled": False, - "site_key": None, + "edit_reasons": [{"code": "test-edit-reason", "label": "Test Edit Reason"}], + "post_close_reasons": [{"code": "test-close-reason", "label": "Test Close Reason"}], + 'show_discussions': True, + 'is_notify_all_learners_enabled': False, + 'captcha_settings': { + 'enabled': False, + 'site_key': None, }, "is_email_verified": True, "only_verified_users_can_post": False, - "content_creation_rate_limited": False, - }, + "content_creation_rate_limited": False + } ) @@ -653,10 +574,8 @@ class RetireViewTest(DiscussionAPIViewTestMixin, ModuleStoreTestCase): def setUp(self): super().setUp() - RetirementState.objects.create(state_name="PENDING", state_execution_order=1) - self.retire_forums_state = RetirementState.objects.create( - state_name="RETIRE_FORUMS", state_execution_order=11 - ) + RetirementState.objects.create(state_name='PENDING', state_execution_order=1) + self.retire_forums_state = RetirementState.objects.create(state_name='RETIRE_FORUMS', state_execution_order=11) self.retirement = UserRetirementStatus.create_retirement(self.user) self.retirement.current_state = self.retire_forums_state @@ -667,8 +586,8 @@ def setUp(self): self.retired_username = get_retired_username_by_username(self.user.username) self.url = reverse("retire_discussion_user") patcher = mock.patch( - "openedx.core.djangoapps.discussions.config.waffle.ENABLE_FORUM_V2.is_enabled", - return_value=False, + 'openedx.core.djangoapps.discussions.config.waffle.ENABLE_FORUM_V2.is_enabled', + return_value=False ) patcher.start() self.addCleanup(patcher.stop) @@ -680,14 +599,14 @@ def assert_response_correct(self, response, expected_status, expected_content): assert response.status_code == expected_status if expected_content: - assert response.content.decode("utf-8") == expected_content + assert response.content.decode('utf-8') == expected_content def build_jwt_headers(self, user): """ Helper function for creating headers for the JWT authentication. """ token = create_jwt_for_user(user) - headers = {"HTTP_AUTHORIZATION": "JWT " + token} + headers = {'HTTP_AUTHORIZATION': 'JWT ' + token} return headers def test_basic(self): @@ -696,7 +615,7 @@ def test_basic(self): """ self.register_get_user_retire_response(self.user) headers = self.build_jwt_headers(self.superuser) - data = {"username": self.user.username} + data = {'username': self.user.username} response = self.superuser_client.post(self.url, data, **headers) self.assert_response_correct(response, 204, b"") @@ -704,11 +623,9 @@ def test_downstream_forums_error(self): """ Check that we bubble up errors from the comments service """ - self.register_get_user_retire_response( - self.user, status=500, body="Server error" - ) + self.register_get_user_retire_response(self.user, status=500, body="Server error") headers = self.build_jwt_headers(self.superuser) - data = {"username": self.user.username} + data = {'username': self.user.username} response = self.superuser_client.post(self.url, data, **headers) self.assert_response_correct(response, 500, '"Server error"') @@ -718,7 +635,7 @@ def test_nonexistent_user(self): """ nonexistent_username = "nonexistent user" self.retired_username = get_retired_username_by_username(nonexistent_username) - data = {"username": nonexistent_username} + data = {'username': nonexistent_username} headers = self.build_jwt_headers(self.superuser) response = self.superuser_client.post(self.url, data, **headers) self.assert_response_correct(response, 404, None) @@ -732,10 +649,7 @@ def test_not_authenticated(self): @ddt.ddt @httpretty.activate -@mock.patch( - "django.conf.settings.USERNAME_REPLACEMENT_WORKER", - "test_replace_username_service_worker", -) +@mock.patch('django.conf.settings.USERNAME_REPLACEMENT_WORKER', 'test_replace_username_service_worker') @mock.patch.dict("django.conf.settings.FEATURES", {"ENABLE_DISCUSSION_SERVICE": True}) class ReplaceUsernamesViewTest(DiscussionAPIViewTestMixin, ModuleStoreTestCase): """Tests for ReplaceUsernamesView""" @@ -748,8 +662,8 @@ def setUp(self): self.new_username = "test_username_replacement" self.url = reverse("replace_discussion_username") patcher = mock.patch( - "openedx.core.djangoapps.discussions.config.waffle.ENABLE_FORUM_V2.is_enabled", - return_value=False, + 'openedx.core.djangoapps.discussions.config.waffle.ENABLE_FORUM_V2.is_enabled', + return_value=False ) patcher.start() self.addCleanup(patcher.stop) @@ -768,28 +682,34 @@ def build_jwt_headers(self, user): Helper function for creating headers for the JWT authentication. """ token = create_jwt_for_user(user) - headers = {"HTTP_AUTHORIZATION": "JWT " + token} + headers = {'HTTP_AUTHORIZATION': 'JWT ' + token} return headers def call_api(self, user, client, data): - """Helper function to call API with data""" + """ Helper function to call API with data """ data = json.dumps(data) headers = self.build_jwt_headers(user) - return client.post(self.url, data, content_type="application/json", **headers) + return client.post(self.url, data, content_type='application/json', **headers) - @ddt.data([{}, {}], {}, [{"test_key": "test_value", "test_key_2": "test_value_2"}]) + @ddt.data( + [{}, {}], + {}, + [{"test_key": "test_value", "test_key_2": "test_value_2"}] + ) def test_bad_schema(self, mapping_data): - """Verify the endpoint rejects bad data schema""" - data = {"username_mappings": mapping_data} + """ Verify the endpoint rejects bad data schema """ + data = { + "username_mappings": mapping_data + } response = self.call_api(self.worker, self.worker_client, data) assert response.status_code == 400 def test_auth(self): - """Verify the endpoint only works with the service worker""" + """ Verify the endpoint only works with the service worker """ data = { "username_mappings": [ {"test_username_1": "test_new_username_1"}, - {"test_username_2": "test_new_username_2"}, + {"test_username_2": "test_new_username_2"} ] } @@ -807,15 +727,15 @@ def test_auth(self): assert response.status_code == 200 def test_basic(self): - """Check successful replacement""" + """ Check successful replacement """ data = { "username_mappings": [ {self.user.username: self.new_username}, ] } expected_response = { - "failed_replacements": [], - "successful_replacements": data["username_mappings"], + 'failed_replacements': [], + 'successful_replacements': data["username_mappings"] } self.register_get_username_replacement_response(self.user) response = self.call_api(self.worker, self.worker_client, data) @@ -831,9 +751,7 @@ def test_not_authenticated(self): @ddt.ddt @mock.patch.dict("django.conf.settings.FEATURES", {"ENABLE_DISCUSSION_SERVICE": True}) -class CourseTopicsViewTest( - DiscussionAPIViewTestMixin, CommentsServiceMockMixin, ModuleStoreTestCase -): +class CourseTopicsViewTest(DiscussionAPIViewTestMixin, CommentsServiceMockMixin, ModuleStoreTestCase): """ Tests for CourseTopicsView """ @@ -850,12 +768,10 @@ def setUp(self): "courseware-2": {"discussion": 4, "question": 5}, "courseware-3": {"discussion": 7, "question": 2}, } - self.register_get_course_commentable_counts_response( - self.course.id, self.thread_counts_map - ) + self.register_get_course_commentable_counts_response(self.course.id, self.thread_counts_map) patcher = mock.patch( - "openedx.core.djangoapps.discussions.config.waffle.ENABLE_FORUM_V2.is_enabled", - return_value=False, + 'openedx.core.djangoapps.discussions.config.waffle.ENABLE_FORUM_V2.is_enabled', + return_value=False ) patcher.start() self.addCleanup(patcher.stop) @@ -870,7 +786,7 @@ def create_course(self, blocks_count, module_store, topics): run="c", start=datetime.now(UTC), default_store=module_store, - discussion_topics=topics, + discussion_topics=topics ) CourseEnrollmentFactory.create(user=self.user, course_id=course.id) course_url = reverse("course_topics", kwargs={"course_id": str(course.id)}) @@ -878,10 +794,10 @@ def create_course(self, blocks_count, module_store, topics): for i in range(blocks_count): BlockFactory.create( parent_location=course.location, - category="discussion", - discussion_id=f"id_module_{i}", - discussion_category=f"Category {i}", - discussion_target=f"Discussion {i}", + category='discussion', + discussion_id=f'id_module_{i}', + discussion_category=f'Category {i}', + discussion_target=f'Discussion {i}', publish_item=False, ) return course_url, course.id @@ -896,7 +812,7 @@ def make_discussion_xblock(self, topic_id, category, subcategory, **kwargs): discussion_id=topic_id, discussion_category=category, discussion_target=subcategory, - **kwargs, + **kwargs ) def test_404(self): @@ -904,7 +820,9 @@ def test_404(self): reverse("course_topics", kwargs={"course_id": "non/existent/course"}) ) self.assert_response_correct( - response, 404, {"developer_message": "Course not found."} + response, + 404, + {"developer_message": "Course not found."} ) def test_basic(self): @@ -914,30 +832,21 @@ def test_basic(self): 200, { "courseware_topics": [], - "non_courseware_topics": [ - { - "id": "test_topic", - "name": "Test Topic", - "children": [], - "thread_list_url": "http://testserver/api/discussion/v1/threads/" - "?course_id=course-v1%3Ax%2By%2Bz&topic_id=test_topic", - "thread_counts": {"discussion": 0, "question": 0}, - } - ], - }, + "non_courseware_topics": [{ + "id": "test_topic", + "name": "Test Topic", + "children": [], + "thread_list_url": 'http://testserver/api/discussion/v1/threads/' + '?course_id=course-v1%3Ax%2By%2Bz&topic_id=test_topic', + "thread_counts": {"discussion": 0, "question": 0}, + }], + } ) @ddt.data( (2, ModuleStoreEnum.Type.split, 2, {"Test Topic 1": {"id": "test_topic_1"}}), - ( - 2, - ModuleStoreEnum.Type.split, - 2, - { - "Test Topic 1": {"id": "test_topic_1"}, - "Test Topic 2": {"id": "test_topic_2"}, - }, - ), + (2, ModuleStoreEnum.Type.split, 2, + {"Test Topic 1": {"id": "test_topic_1"}, "Test Topic 2": {"id": "test_topic_2"}}), (10, ModuleStoreEnum.Type.split, 2, {"Test Topic 1": {"id": "test_topic_1"}}), ) @ddt.unpack @@ -959,7 +868,7 @@ def test_discussion_topic_404(self): self.assert_response_correct( response, 404, - {"developer_message": "Discussion not found for 'invalid_topic_id'."}, + {"developer_message": "Discussion not found for 'invalid_topic_id'."} ) def test_topic_id(self): @@ -979,41 +888,38 @@ def test_topic_id(self): "non_courseware_topics": [], "courseware_topics": [ { - "children": [ - { - "children": [], - "id": "topic_id_1", - "thread_list_url": "http://testserver/api/discussion/v1/threads/?" - "course_id=course-v1%3Ax%2By%2Bz&topic_id=topic_id_1", - "name": "test_target_1", - "thread_counts": {"discussion": 0, "question": 0}, - } - ], + "children": [{ + "children": [], + "id": "topic_id_1", + "thread_list_url": "http://testserver/api/discussion/v1/threads/?" + "course_id=course-v1%3Ax%2By%2Bz&topic_id=topic_id_1", + "name": "test_target_1", + "thread_counts": {"discussion": 0, "question": 0}, + }], "id": None, "thread_list_url": "http://testserver/api/discussion/v1/threads/?" - "course_id=course-v1%3Ax%2By%2Bz&topic_id=topic_id_1", + "course_id=course-v1%3Ax%2By%2Bz&topic_id=topic_id_1", "name": "test_category_1", "thread_counts": None, }, { - "children": [ - { + "children": + [{ "children": [], "id": "topic_id_2", "thread_list_url": "http://testserver/api/discussion/v1/threads/?" - "course_id=course-v1%3Ax%2By%2Bz&topic_id=topic_id_2", + "course_id=course-v1%3Ax%2By%2Bz&topic_id=topic_id_2", "name": "test_target_2", "thread_counts": {"discussion": 0, "question": 0}, - } - ], + }], "id": None, "thread_list_url": "http://testserver/api/discussion/v1/threads/?" - "course_id=course-v1%3Ax%2By%2Bz&topic_id=topic_id_2", + "course_id=course-v1%3Ax%2By%2Bz&topic_id=topic_id_2", "name": "test_category_2", "thread_counts": None, - }, - ], - }, + } + ] + } ) @override_waffle_flag(ENABLE_NEW_STRUCTURE_DISCUSSIONS, True) @@ -1024,46 +930,45 @@ def test_new_course_structure_response(self): """ chapter = BlockFactory.create( parent_location=self.course.location, - category="chapter", + category='chapter', display_name="Week 1", start=datetime(2015, 3, 1, tzinfo=UTC), ) sequential = BlockFactory.create( parent_location=chapter.location, - category="sequential", + category='sequential', display_name="Lesson 1", start=datetime(2015, 3, 1, tzinfo=UTC), ) BlockFactory.create( parent_location=sequential.location, - category="vertical", - display_name="vertical", + category='vertical', + display_name='vertical', start=datetime(2015, 4, 1, tzinfo=UTC), ) DiscussionsConfiguration.objects.create( - context_key=self.course.id, provider_type=Provider.OPEN_EDX + context_key=self.course.id, + provider_type=Provider.OPEN_EDX ) update_discussions_settings_from_course_task(str(self.course.id)) response = json.loads(self.client.get(self.url).content.decode()) - keys = ["children", "id", "name", "thread_counts", "thread_list_url"] - assert list(response.keys()) == ["courseware_topics", "non_courseware_topics"] - assert len(response["courseware_topics"]) == 1 - courseware_keys = list(response["courseware_topics"][0].keys()) + keys = ['children', 'id', 'name', 'thread_counts', 'thread_list_url'] + assert list(response.keys()) == ['courseware_topics', 'non_courseware_topics'] + assert len(response['courseware_topics']) == 1 + courseware_keys = list(response['courseware_topics'][0].keys()) courseware_keys.sort() assert courseware_keys == keys - assert len(response["non_courseware_topics"]) == 1 - non_courseware_keys = list(response["non_courseware_topics"][0].keys()) + assert len(response['non_courseware_topics']) == 1 + non_courseware_keys = list(response['non_courseware_topics'][0].keys()) non_courseware_keys.sort() assert non_courseware_keys == keys @ddt.ddt -@mock.patch("lms.djangoapps.discussion.rest_api.api._get_course", mock.Mock()) +@mock.patch('lms.djangoapps.discussion.rest_api.api._get_course', mock.Mock()) @mock.patch.dict("django.conf.settings.FEATURES", {"ENABLE_DISCUSSION_SERVICE": True}) @override_waffle_flag(ENABLE_NEW_STRUCTURE_DISCUSSIONS, True) -class CourseTopicsViewV3Test( - DiscussionAPIViewTestMixin, CommentsServiceMockMixin, ModuleStoreTestCase -): +class CourseTopicsViewV3Test(DiscussionAPIViewTestMixin, CommentsServiceMockMixin, ModuleStoreTestCase): """ Tests for CourseTopicsViewV3 """ @@ -1079,68 +984,55 @@ def setUp(self) -> None: end=datetime(2028, 1, 1), enrollment_start=datetime(2020, 1, 1), enrollment_end=datetime(2028, 1, 1), - discussion_topics={ - "Course Wide Topic": { - "id": "course-wide-topic", - "usage_key": None, - } - }, + discussion_topics={"Course Wide Topic": { + "id": 'course-wide-topic', + "usage_key": None, + }} ) self.chapter = BlockFactory.create( parent_location=self.course.location, - category="chapter", + category='chapter', display_name="Week 1", start=datetime(2015, 3, 1, tzinfo=UTC), ) self.sequential = BlockFactory.create( parent_location=self.chapter.location, - category="sequential", + category='sequential', display_name="Lesson 1", start=datetime(2015, 3, 1, tzinfo=UTC), ) self.verticals = [ BlockFactory.create( parent_location=self.sequential.location, - category="vertical", - display_name="vertical", + category='vertical', + display_name='vertical', start=datetime(2015, 4, 1, tzinfo=UTC), ) ] course_key = self.course.id - self.config = DiscussionsConfiguration.objects.create( - context_key=course_key, provider_type=Provider.OPEN_EDX - ) + self.config = DiscussionsConfiguration.objects.create(context_key=course_key, provider_type=Provider.OPEN_EDX) topic_links = [] update_discussions_settings_from_course_task(str(course_key)) - topic_id_query = DiscussionTopicLink.objects.filter( - context_key=course_key - ).values_list( - "external_id", - flat=True, + topic_id_query = DiscussionTopicLink.objects.filter(context_key=course_key).values_list( + 'external_id', flat=True, ) - topic_ids = list(topic_id_query.order_by("ordering")) + topic_ids = list(topic_id_query.order_by('ordering')) DiscussionTopicLink.objects.bulk_create(topic_links) self.topic_stats = { - **{ - topic_id: dict( - discussion=random.randint(0, 10), question=random.randint(0, 10) - ) - for topic_id in set(topic_ids) - }, + **{topic_id: dict(discussion=random.randint(0, 10), question=random.randint(0, 10)) + for topic_id in set(topic_ids)}, topic_ids[0]: dict(discussion=0, question=0), } patcher = mock.patch( - "lms.djangoapps.discussion.rest_api.api.get_course_commentable_counts", + 'lms.djangoapps.discussion.rest_api.api.get_course_commentable_counts', mock.Mock(return_value=self.topic_stats), ) patcher.start() self.addCleanup(patcher.stop) - self.url = reverse( - "course_topics_v3", kwargs={"course_id": str(self.course.id)} - ) + self.url = reverse("course_topics_v3", kwargs={"course_id": str(self.course.id)}) patcher = mock.patch( - "openedx.core.djangoapps.discussions.config.waffle.ENABLE_FORUM_V2.is_enabled", - return_value=False, + 'openedx.core.djangoapps.discussions.config.waffle.ENABLE_FORUM_V2.is_enabled', + return_value=False ) patcher.start() self.addCleanup(patcher.stop) @@ -1149,23 +1041,12 @@ def test_basic(self): response = self.client.get(self.url) data = json.loads(response.content.decode()) expected_non_courseware_keys = [ - "id", - "usage_key", - "name", - "thread_counts", - "enabled_in_context", - "courseware", + 'id', 'usage_key', 'name', 'thread_counts', 'enabled_in_context', + 'courseware' ] expected_courseware_keys = [ - "id", - "block_id", - "lms_web_url", - "legacy_web_url", - "student_view_url", - "type", - "display_name", - "children", - "courseware", + 'id', 'block_id', 'lms_web_url', 'legacy_web_url', 'student_view_url', + 'type', 'display_name', 'children', 'courseware' ] assert response.status_code == 200 assert len(data) == 2 @@ -1173,11 +1054,11 @@ def test_basic(self): assert non_courseware_topic_keys == expected_non_courseware_keys courseware_topic_keys = list(data[1].keys()) assert courseware_topic_keys == expected_courseware_keys - expected_courseware_keys.remove("courseware") - sequential_keys = list(data[1]["children"][0].keys()) - assert sequential_keys == (expected_courseware_keys + ["thread_counts"]) - expected_non_courseware_keys.remove("courseware") - vertical_keys = list(data[1]["children"][0]["children"][0].keys()) + expected_courseware_keys.remove('courseware') + sequential_keys = list(data[1]['children'][0].keys()) + assert sequential_keys == (expected_courseware_keys + ['thread_counts']) + expected_non_courseware_keys.remove('courseware') + vertical_keys = list(data[1]['children'][0]['children'][0].keys()) assert vertical_keys == expected_non_courseware_keys @@ -1218,21 +1099,14 @@ def setUp(self): {"key": "close_reason", "value": None}, { "key": "comment_list_url", - "value": "http://testserver/api/discussion/v1/comments/?thread_id=test_thread", + "value": "http://testserver/api/discussion/v1/comments/?thread_id=test_thread" }, { "key": "editable_fields", "value": [ - "abuse_flagged", - "anonymous", - "copy_link", - "following", - "raw_body", - "read", - "title", - "topic_id", - "type", - ], + 'abuse_flagged', 'anonymous', 'copy_link', 'following', 'raw_body', + 'read', 'title', 'topic_id', 'type' + ] }, {"key": "endorsed_comment_list_url", "value": None}, {"key": "following", "value": False}, @@ -1243,39 +1117,32 @@ def setUp(self): {"key": "non_endorsed_comment_list_url", "value": None}, {"key": "preview_body", "value": "Test body"}, {"key": "raw_body", "value": "Test body"}, + {"key": "rendered_body", "value": "

    Test body

    "}, {"key": "response_count", "value": 0}, {"key": "topic_id", "value": "test_topic"}, {"key": "type", "value": "discussion"}, - { - "key": "users", - "value": { - self.user.username: { - "profile": { - "image": { - "has_image": False, - "image_url_full": "http://testserver/static/default_500.png", - "image_url_large": "http://testserver/static/default_120.png", - "image_url_medium": "http://testserver/static/default_50.png", - "image_url_small": "http://testserver/static/default_30.png", - } + {"key": "users", "value": { + self.user.username: { + "profile": { + "image": { + "has_image": False, + "image_url_full": "http://testserver/static/default_500.png", + "image_url_large": "http://testserver/static/default_120.png", + "image_url_medium": "http://testserver/static/default_50.png", + "image_url_small": "http://testserver/static/default_30.png", } } - }, - }, + } + }}, {"key": "vote_count", "value": 4}, {"key": "voted", "value": False}, - {"key": "is_deleted", "value": None}, - {"key": "deleted_at", "value": None}, - {"key": "deleted_by", "value": None}, - {"key": "deleted_by_label", "value": None}, + ] - self.url = reverse( - "discussion_learner_threads", kwargs={"course_id": str(self.course.id)} - ) + self.url = reverse("discussion_learner_threads", kwargs={'course_id': str(self.course.id)}) patcher = mock.patch( - "openedx.core.djangoapps.discussions.config.waffle.ENABLE_FORUM_V2.is_enabled", - return_value=False, + 'openedx.core.djangoapps.discussions.config.waffle.ENABLE_FORUM_V2.is_enabled', + return_value=False ) patcher.start() self.addCleanup(patcher.stop) @@ -1286,12 +1153,12 @@ def update_thread(self, thread): Value of these keys has been defined in setUp function """ for element in self.add_keys: - thread[element["key"]] = element["value"] + thread[element['key']] = element['value'] for pair in self.replace_keys: - thread[pair["to"]] = thread.pop(pair["from"]) + thread[pair['to']] = thread.pop(pair['from']) for key in self.remove_keys: thread.pop(key) - thread["comment_count"] += 1 + thread['comment_count'] += 1 return thread def test_basic(self): @@ -1303,26 +1170,22 @@ def test_basic(self): """ self.register_get_user_response(self.user) expected_cs_comments_response = { - "collection": [ - make_minimal_cs_thread( - { - "id": "test_thread", - "course_id": str(self.course.id), - "commentable_id": "test_topic", - "user_id": str(self.user.id), - "username": self.user.username, - "created_at": "2015-04-28T00:00:00Z", - "updated_at": "2015-04-28T11:11:11Z", - "title": "Test Title", - "body": "Test body", - "votes": {"up_count": 4}, - "comments_count": 5, - "unread_comments_count": 3, - "closed_by_label": None, - "edit_by_label": None, - } - ) - ], + "collection": [make_minimal_cs_thread({ + "id": "test_thread", + "course_id": str(self.course.id), + "commentable_id": "test_topic", + "user_id": str(self.user.id), + "username": self.user.username, + "created_at": "2015-04-28T00:00:00Z", + "updated_at": "2015-04-28T11:11:11Z", + "title": "Test Title", + "body": "Test body", + "votes": {"up_count": 4}, + "comments_count": 5, + "unread_comments_count": 3, + "closed_by_label": None, + "edit_by_label": None, + })], "page": 1, "num_pages": 1, } @@ -1330,14 +1193,14 @@ def test_basic(self): self.url += f"?username={self.user.username}" response = self.client.get(self.url) assert response.status_code == 200 - response_data = json.loads(response.content.decode("utf-8")) - expected_api_response = expected_cs_comments_response["collection"] + response_data = json.loads(response.content.decode('utf-8')) + expected_api_response = expected_cs_comments_response['collection'] for thread in expected_api_response: self.update_thread(thread) - assert response_data["results"] == expected_api_response - assert response_data["pagination"] == { + assert response_data['results'] == expected_api_response + assert response_data['pagination'] == { "next": None, "previous": None, "count": 1, @@ -1367,24 +1230,20 @@ def test_thread_type_by(self, thread_type): thread_type (str): Value of thread_type can be 'None', 'discussion' and 'question' """ - threads = [ - make_minimal_cs_thread( - { - "id": "test_thread", - "course_id": str(self.course.id), - "commentable_id": "test_topic", - "user_id": str(self.user.id), - "username": self.user.username, - "created_at": "2015-04-28T00:00:00Z", - "updated_at": "2015-04-28T11:11:11Z", - "title": "Test Title", - "body": "Test body", - "votes": {"up_count": 4}, - "comments_count": 5, - "unread_comments_count": 3, - } - ) - ] + threads = [make_minimal_cs_thread({ + "id": "test_thread", + "course_id": str(self.course.id), + "commentable_id": "test_topic", + "user_id": str(self.user.id), + "username": self.user.username, + "created_at": "2015-04-28T00:00:00Z", + "updated_at": "2015-04-28T11:11:11Z", + "title": "Test Title", + "body": "Test body", + "votes": {"up_count": 4}, + "comments_count": 5, + "unread_comments_count": 3, + })] expected_cs_comments_response = { "collection": threads, "page": 1, @@ -1398,26 +1257,23 @@ def test_thread_type_by(self, thread_type): "course_id": str(self.course.id), "username": self.user.username, "thread_type": thread_type, - }, - ) - assert response.status_code == 200 - self.assert_last_query_params( - { - "user_id": [str(self.user.id)], - "course_id": [str(self.course.id)], - "page": ["1"], - "per_page": ["10"], - "thread_type": [thread_type], - "sort_key": ["activity"], - "count_flagged": ["False"], - "show_deleted": ["False"], } ) + assert response.status_code == 200 + self.assert_last_query_params({ + "user_id": [str(self.user.id)], + "course_id": [str(self.course.id)], + "page": ["1"], + "per_page": ["10"], + "thread_type": [thread_type], + "sort_key": ['activity'], + "count_flagged": ["False"] + }) @ddt.data( ("last_activity_at", "activity"), ("comment_count", "comments"), - ("vote_count", "votes"), + ("vote_count", "votes") ) @ddt.unpack def test_order_by(self, http_query, cc_query): @@ -1428,24 +1284,20 @@ def test_order_by(self, http_query, cc_query): http_query (str): Query string sent in the http request cc_query (str): Query string used for the comments client service """ - threads = [ - make_minimal_cs_thread( - { - "id": "test_thread", - "course_id": str(self.course.id), - "commentable_id": "test_topic", - "user_id": str(self.user.id), - "username": self.user.username, - "created_at": "2015-04-28T00:00:00Z", - "updated_at": "2015-04-28T11:11:11Z", - "title": "Test Title", - "body": "Test body", - "votes": {"up_count": 4}, - "comments_count": 5, - "unread_comments_count": 3, - } - ) - ] + threads = [make_minimal_cs_thread({ + "id": "test_thread", + "course_id": str(self.course.id), + "commentable_id": "test_topic", + "user_id": str(self.user.id), + "username": self.user.username, + "created_at": "2015-04-28T00:00:00Z", + "updated_at": "2015-04-28T11:11:11Z", + "title": "Test Title", + "body": "Test body", + "votes": {"up_count": 4}, + "comments_count": 5, + "unread_comments_count": 3, + })] expected_cs_comments_response = { "collection": threads, "page": 1, @@ -1459,20 +1311,17 @@ def test_order_by(self, http_query, cc_query): "course_id": str(self.course.id), "username": self.user.username, "order_by": http_query, - }, - ) - assert response.status_code == 200 - self.assert_last_query_params( - { - "user_id": [str(self.user.id)], - "course_id": [str(self.course.id)], - "page": ["1"], - "per_page": ["10"], - "sort_key": [cc_query], - "count_flagged": ["False"], - "show_deleted": ["False"], } ) + assert response.status_code == 200 + self.assert_last_query_params({ + "user_id": [str(self.user.id)], + "course_id": [str(self.course.id)], + "page": ["1"], + "per_page": ["10"], + "sort_key": [cc_query], + "count_flagged": ["False"] + }) @ddt.data("flagged", "unanswered", "unread", "unresponded") def test_status_by(self, post_status): @@ -1483,24 +1332,20 @@ def test_status_by(self, post_status): post_status (str): Value of post_status can be 'flagged', 'unanswered' and 'unread' """ - threads = [ - make_minimal_cs_thread( - { - "id": "test_thread", - "course_id": str(self.course.id), - "commentable_id": "test_topic", - "user_id": str(self.user.id), - "username": self.user.username, - "created_at": "2015-04-28T00:00:00Z", - "updated_at": "2015-04-28T11:11:11Z", - "title": "Test Title", - "body": "Test body", - "votes": {"up_count": 4}, - "comments_count": 5, - "unread_comments_count": 3, - } - ) - ] + threads = [make_minimal_cs_thread({ + "id": "test_thread", + "course_id": str(self.course.id), + "commentable_id": "test_topic", + "user_id": str(self.user.id), + "username": self.user.username, + "created_at": "2015-04-28T00:00:00Z", + "updated_at": "2015-04-28T11:11:11Z", + "title": "Test Title", + "body": "Test body", + "votes": {"up_count": 4}, + "comments_count": 5, + "unread_comments_count": 3, + })] expected_cs_comments_response = { "collection": threads, "page": 1, @@ -1514,37 +1359,29 @@ def test_status_by(self, post_status): "course_id": str(self.course.id), "username": self.user.username, "status": post_status, - }, + } ) if post_status == "flagged": assert response.status_code == 403 else: assert response.status_code == 200 - self.assert_last_query_params( - { - "user_id": [str(self.user.id)], - "course_id": [str(self.course.id)], - "page": ["1"], - "per_page": ["10"], - post_status: ["True"], - "sort_key": ["activity"], - "count_flagged": ["False"], - "show_deleted": ["False"], - } - ) + self.assert_last_query_params({ + "user_id": [str(self.user.id)], + "course_id": [str(self.course.id)], + "page": ["1"], + "per_page": ["10"], + post_status: ['True'], + "sort_key": ['activity'], + "count_flagged": ["False"] + }) @ddt.ddt -class CourseDiscussionSettingsAPIViewTest( - APITestCase, UrlResetMixin, ModuleStoreTestCase -): +class CourseDiscussionSettingsAPIViewTest(APITestCase, UrlResetMixin, ModuleStoreTestCase): """ Test the course discussion settings handler API endpoint. """ - - @mock.patch.dict( - "django.conf.settings.FEATURES", {"ENABLE_DISCUSSION_SERVICE": True} - ) + @mock.patch.dict("django.conf.settings.FEATURES", {"ENABLE_DISCUSSION_SERVICE": True}) def setUp(self): super().setUp() self.course = CourseFactory.create( @@ -1552,26 +1389,24 @@ def setUp(self): course="y", run="z", start=datetime.now(UTC), - discussion_topics={"Test Topic": {"id": "test_topic"}}, - ) - self.path = reverse( - "discussion_course_settings", kwargs={"course_id": str(self.course.id)} + discussion_topics={"Test Topic": {"id": "test_topic"}} ) + self.path = reverse('discussion_course_settings', kwargs={'course_id': str(self.course.id)}) self.password = self.TEST_PASSWORD - self.user = UserFactory(username="staff", password=self.password, is_staff=True) + self.user = UserFactory(username='staff', password=self.password, is_staff=True) patcher = mock.patch( - "openedx.core.djangoapps.discussions.config.waffle.ENABLE_FORUM_V2.is_enabled", - return_value=False, + 'openedx.core.djangoapps.discussions.config.waffle.ENABLE_FORUM_V2.is_enabled', + return_value=False ) patcher.start() self.addCleanup(patcher.stop) def _get_oauth_headers(self, user): """Return the OAuth headers for testing OAuth authentication""" - access_token = AccessTokenFactory.create( - user=user, application=ApplicationFactory() - ).token - headers = {"HTTP_AUTHORIZATION": "Bearer " + access_token} + access_token = AccessTokenFactory.create(user=user, application=ApplicationFactory()).token + headers = { + 'HTTP_AUTHORIZATION': 'Bearer ' + access_token + } return headers def _login_as_staff(self): @@ -1579,30 +1414,24 @@ def _login_as_staff(self): self.client.login(username=self.user.username, password=self.password) def _login_as_discussion_staff(self): - user = UserFactory(username="abc", password="abc") - role = Role.objects.create(name="Administrator", course_id=self.course.id) + user = UserFactory(username='abc', password='abc') + role = Role.objects.create(name='Administrator', course_id=self.course.id) role.users.set([user]) - self.client.login(username=user.username, password="abc") + self.client.login(username=user.username, password='abc') def _create_divided_discussions(self): """Create some divided discussions for testing.""" - divided_inline_discussions = [ - "Topic A", - ] - divided_course_wide_discussions = [ - "Topic B", - ] - divided_discussions = ( - divided_inline_discussions + divided_course_wide_discussions - ) + divided_inline_discussions = ['Topic A', ] + divided_course_wide_discussions = ['Topic B', ] + divided_discussions = divided_inline_discussions + divided_course_wide_discussions BlockFactory.create( parent=self.course, - category="discussion", - discussion_id=topic_name_to_id(self.course, "Topic A"), - discussion_category="Chapter", - discussion_target="Discussion", - start=datetime.now(), + category='discussion', + discussion_id=topic_name_to_id(self.course, 'Topic A'), + discussion_category='Chapter', + discussion_target='Discussion', + start=datetime.now() ) discussion_topics = { "Topic B": {"id": "Topic B"}, @@ -1611,36 +1440,31 @@ def _create_divided_discussions(self): config_course_discussions( self.course, discussion_topics=discussion_topics, - divided_discussions=divided_discussions, + divided_discussions=divided_discussions ) return divided_inline_discussions, divided_course_wide_discussions def _get_expected_response(self): """Return the default expected response before any changes to the discussion settings.""" return { - "always_divide_inline_discussions": False, - "divided_inline_discussions": [], - "divided_course_wide_discussions": [], - "id": 1, - "division_scheme": "cohort", - "available_division_schemes": ["cohort"], - "reported_content_email_notifications": False, + 'always_divide_inline_discussions': False, + 'divided_inline_discussions': [], + 'divided_course_wide_discussions': [], + 'id': 1, + 'division_scheme': 'cohort', + 'available_division_schemes': ['cohort'], + 'reported_content_email_notifications': False, } def patch_request(self, data, headers=None): headers = headers if headers else {} - return self.client.patch( - self.path, - json.dumps(data), - content_type="application/merge-patch+json", - **headers, - ) + return self.client.patch(self.path, json.dumps(data), content_type='application/merge-patch+json', **headers) def _assert_current_settings(self, expected_response): """Validate the current discussion settings against the expected response.""" response = self.client.get(self.path) assert response.status_code == 200 - content = json.loads(response.content.decode("utf-8")) + content = json.loads(response.content.decode('utf-8')) assert content == expected_response def _assert_patched_settings(self, data, expected_response): @@ -1649,7 +1473,7 @@ def _assert_patched_settings(self, data, expected_response): assert response.status_code == 204 self._assert_current_settings(expected_response) - @ddt.data("get", "patch") + @ddt.data('get', 'patch') def test_authentication_required(self, method): """Test and verify that authentication is required for this endpoint.""" self.client.logout() @@ -1657,8 +1481,8 @@ def test_authentication_required(self, method): assert response.status_code == 401 @ddt.data( - {"is_staff": False, "get_status": 403, "put_status": 403}, - {"is_staff": True, "get_status": 200, "put_status": 204}, + {'is_staff': False, 'get_status': 403, 'put_status': 403}, + {'is_staff': True, 'get_status': 200, 'put_status': 204}, ) @ddt.unpack def test_oauth(self, is_staff, get_status, put_status): @@ -1671,7 +1495,7 @@ def test_oauth(self, is_staff, get_status, put_status): assert response.status_code == get_status response = self.patch_request( - {"always_divide_inline_discussions": True}, headers + {'always_divide_inline_discussions': True}, headers ) assert response.status_code == put_status @@ -1679,68 +1503,66 @@ def test_non_existent_course_id(self): """Test the response when this endpoint is passed a non-existent course id.""" self._login_as_staff() response = self.client.get( - reverse( - "discussion_course_settings", kwargs={"course_id": "course-v1:a+b+c"} - ) + reverse('discussion_course_settings', kwargs={ + 'course_id': 'course-v1:a+b+c' + }) ) assert response.status_code == 404 def test_patch_request_by_discussion_staff(self): """Test the response when patch request is sent by a user with discussions staff role.""" self._login_as_discussion_staff() - response = self.patch_request({"always_divide_inline_discussions": True}) + response = self.patch_request( + {'always_divide_inline_discussions': True} + ) assert response.status_code == 403 def test_get_request_by_discussion_staff(self): """Test the response when get request is sent by a user with discussions staff role.""" self._login_as_discussion_staff() - divided_inline_discussions, divided_course_wide_discussions = ( - self._create_divided_discussions() - ) + divided_inline_discussions, divided_course_wide_discussions = self._create_divided_discussions() response = self.client.get(self.path) assert response.status_code == 200 expected_response = self._get_expected_response() - expected_response["divided_course_wide_discussions"] = [ - topic_name_to_id(self.course, name) - for name in divided_course_wide_discussions + expected_response['divided_course_wide_discussions'] = [ + topic_name_to_id(self.course, name) for name in divided_course_wide_discussions ] - expected_response["divided_inline_discussions"] = [ + expected_response['divided_inline_discussions'] = [ topic_name_to_id(self.course, name) for name in divided_inline_discussions ] - content = json.loads(response.content.decode("utf-8")) + content = json.loads(response.content.decode('utf-8')) assert content == expected_response def test_get_request_by_non_staff_user(self): """Test the response when get request is sent by a regular user with no staff role.""" - user = UserFactory(username="abc", password="abc") - self.client.login(username=user.username, password="abc") + user = UserFactory(username='abc', password='abc') + self.client.login(username=user.username, password='abc') response = self.client.get(self.path) assert response.status_code == 403 def test_patch_request_by_non_staff_user(self): """Test the response when patch request is sent by a regular user with no staff role.""" - user = UserFactory(username="abc", password="abc") - self.client.login(username=user.username, password="abc") - response = self.patch_request({"always_divide_inline_discussions": True}) + user = UserFactory(username='abc', password='abc') + self.client.login(username=user.username, password='abc') + response = self.patch_request( + {'always_divide_inline_discussions': True} + ) assert response.status_code == 403 def test_get_settings(self): """Test the current discussion settings against the expected response.""" - divided_inline_discussions, divided_course_wide_discussions = ( - self._create_divided_discussions() - ) + divided_inline_discussions, divided_course_wide_discussions = self._create_divided_discussions() self._login_as_staff() response = self.client.get(self.path) assert response.status_code == 200 expected_response = self._get_expected_response() - expected_response["divided_course_wide_discussions"] = [ - topic_name_to_id(self.course, name) - for name in divided_course_wide_discussions + expected_response['divided_course_wide_discussions'] = [ + topic_name_to_id(self.course, name) for name in divided_course_wide_discussions ] - expected_response["divided_inline_discussions"] = [ + expected_response['divided_inline_discussions'] = [ topic_name_to_id(self.course, name) for name in divided_inline_discussions ] - content = json.loads(response.content.decode("utf-8")) + content = json.loads(response.content.decode('utf-8')) assert content == expected_response def test_available_schemes(self): @@ -1748,23 +1570,18 @@ def test_available_schemes(self): config_course_cohorts(self.course, is_cohorted=False) self._login_as_staff() expected_response = self._get_expected_response() - expected_response["available_division_schemes"] = [] + expected_response['available_division_schemes'] = [] self._assert_current_settings(expected_response) CourseModeFactory.create(course_id=self.course.id, mode_slug=CourseMode.AUDIT) - CourseModeFactory.create( - course_id=self.course.id, mode_slug=CourseMode.VERIFIED - ) + CourseModeFactory.create(course_id=self.course.id, mode_slug=CourseMode.VERIFIED) - expected_response["available_division_schemes"] = [ - CourseDiscussionSettings.ENROLLMENT_TRACK - ] + expected_response['available_division_schemes'] = [CourseDiscussionSettings.ENROLLMENT_TRACK] self._assert_current_settings(expected_response) config_course_cohorts(self.course, is_cohorted=True) - expected_response["available_division_schemes"] = [ - CourseDiscussionSettings.COHORT, - CourseDiscussionSettings.ENROLLMENT_TRACK, + expected_response['available_division_schemes'] = [ + CourseDiscussionSettings.COHORT, CourseDiscussionSettings.ENROLLMENT_TRACK ] self._assert_current_settings(expected_response) @@ -1778,11 +1595,11 @@ def test_empty_body_patch_request(self): assert response.status_code == 400 @ddt.data( - {"abc": 123}, - {"divided_course_wide_discussions": 3}, - {"divided_inline_discussions": "a"}, - {"always_divide_inline_discussions": ["a"]}, - {"division_scheme": True}, + {'abc': 123}, + {'divided_course_wide_discussions': 3}, + {'divided_inline_discussions': 'a'}, + {'always_divide_inline_discussions': ['a']}, + {'division_scheme': True} ) def test_invalid_body_parameters(self, body): """Test the response status code on sending a PATCH request with parameters having incorrect types.""" @@ -1796,34 +1613,31 @@ def test_update_always_divide_inline_discussion_settings(self): self._login_as_staff() expected_response = self._get_expected_response() self._assert_current_settings(expected_response) - expected_response["always_divide_inline_discussions"] = True + expected_response['always_divide_inline_discussions'] = True - self._assert_patched_settings( - {"always_divide_inline_discussions": True}, expected_response - ) + self._assert_patched_settings({'always_divide_inline_discussions': True}, expected_response) def test_update_course_wide_discussion_settings(self): """Test whether the 'divided_course_wide_discussions' setting is updated.""" - discussion_topics = {"Topic B": {"id": "Topic B"}} + discussion_topics = { + 'Topic B': {'id': 'Topic B'} + } config_course_cohorts(self.course, is_cohorted=True) config_course_discussions(self.course, discussion_topics=discussion_topics) expected_response = self._get_expected_response() self._login_as_staff() self._assert_current_settings(expected_response) - expected_response["divided_course_wide_discussions"] = [ + expected_response['divided_course_wide_discussions'] = [ topic_name_to_id(self.course, "Topic B") ] self._assert_patched_settings( - { - "divided_course_wide_discussions": [ - topic_name_to_id(self.course, "Topic B") - ] - }, - expected_response, + {'divided_course_wide_discussions': [topic_name_to_id(self.course, "Topic B")]}, + expected_response ) - expected_response["divided_course_wide_discussions"] = [] + expected_response['divided_course_wide_discussions'] = [] self._assert_patched_settings( - {"divided_course_wide_discussions": []}, expected_response + {'divided_course_wide_discussions': []}, + expected_response ) def test_update_inline_discussion_settings(self): @@ -1836,23 +1650,17 @@ def test_update_inline_discussion_settings(self): now = datetime.now() BlockFactory.create( parent_location=self.course.location, - category="discussion", - discussion_id="Topic_A", - discussion_category="Chapter", - discussion_target="Discussion", - start=now, - ) - expected_response["divided_inline_discussions"] = [ - "Topic_A", - ] - self._assert_patched_settings( - {"divided_inline_discussions": ["Topic_A"]}, expected_response + category='discussion', + discussion_id='Topic_A', + discussion_category='Chapter', + discussion_target='Discussion', + start=now ) + expected_response['divided_inline_discussions'] = ['Topic_A', ] + self._assert_patched_settings({'divided_inline_discussions': ['Topic_A']}, expected_response) - expected_response["divided_inline_discussions"] = [] - self._assert_patched_settings( - {"divided_inline_discussions": []}, expected_response - ) + expected_response['divided_inline_discussions'] = [] + self._assert_patched_settings({'divided_inline_discussions': []}, expected_response) def test_update_division_scheme(self): """Test whether the 'division_scheme' setting is updated.""" @@ -1860,17 +1668,15 @@ def test_update_division_scheme(self): self._login_as_staff() expected_response = self._get_expected_response() self._assert_current_settings(expected_response) - expected_response["division_scheme"] = "none" - self._assert_patched_settings({"division_scheme": "none"}, expected_response) + expected_response['division_scheme'] = 'none' + self._assert_patched_settings({'division_scheme': 'none'}, expected_response) def test_update_reported_content_email_notifications(self): """Test whether the 'reported_content_email_notifications' setting is updated.""" config_course_cohorts(self.course, is_cohorted=True) - config_course_discussions( - self.course, reported_content_email_notifications=True - ) + config_course_discussions(self.course, reported_content_email_notifications=True) expected_response = self._get_expected_response() - expected_response["reported_content_email_notifications"] = True + expected_response['reported_content_email_notifications'] = True self._login_as_staff() self._assert_current_settings(expected_response) @@ -1880,15 +1686,12 @@ class CourseDiscussionRolesAPIViewTest(APITestCase, UrlResetMixin, ModuleStoreTe """ Test the course discussion roles management endpoint. """ - - @mock.patch.dict( - "django.conf.settings.FEATURES", {"ENABLE_DISCUSSION_SERVICE": True} - ) + @mock.patch.dict("django.conf.settings.FEATURES", {"ENABLE_DISCUSSION_SERVICE": True}) def setUp(self): super().setUp() patcher = mock.patch( - "openedx.core.djangoapps.discussions.config.waffle.ENABLE_FORUM_V2.is_enabled", - return_value=False, + 'openedx.core.djangoapps.discussions.config.waffle.ENABLE_FORUM_V2.is_enabled', + return_value=False ) patcher.start() self.addCleanup(patcher.stop) @@ -1899,27 +1702,26 @@ def setUp(self): start=datetime.now(UTC), ) self.password = self.TEST_PASSWORD - self.user = UserFactory(username="staff", password=self.password, is_staff=True) - course_key = CourseKey.from_string("course-v1:x+y+z") + self.user = UserFactory(username='staff', password=self.password, is_staff=True) + course_key = CourseKey.from_string('course-v1:x+y+z') seed_permissions_roles(course_key) - @mock.patch.dict( - "django.conf.settings.FEATURES", {"ENABLE_DISCUSSION_SERVICE": True} - ) + @mock.patch.dict("django.conf.settings.FEATURES", {"ENABLE_DISCUSSION_SERVICE": True}) def path(self, course_id=None, role=None): """Return the URL path to the endpoint based on the provided arguments.""" course_id = str(self.course.id) if course_id is None else course_id - role = "Moderator" if role is None else role + role = 'Moderator' if role is None else role return reverse( - "discussion_course_roles", kwargs={"course_id": course_id, "rolename": role} + 'discussion_course_roles', + kwargs={'course_id': course_id, 'rolename': role} ) def _get_oauth_headers(self, user): """Return the OAuth headers for testing OAuth authentication.""" - access_token = AccessTokenFactory.create( - user=user, application=ApplicationFactory() - ).token - headers = {"HTTP_AUTHORIZATION": "Bearer " + access_token} + access_token = AccessTokenFactory.create(user=user, application=ApplicationFactory()).token + headers = { + 'HTTP_AUTHORIZATION': 'Bearer ' + access_token + } return headers def _login_as_staff(self): @@ -1944,11 +1746,9 @@ def _add_users_to_role(self, users, rolename): def post(self, role, user_id, action): """Make a POST request to the endpoint using the provided parameters.""" self._login_as_staff() - return self.client.post( - self.path(role=role), {"user_id": user_id, "action": action} - ) + return self.client.post(self.path(role=role), {'user_id': user_id, 'action': action}) - @ddt.data("get", "post") + @ddt.data('get', 'post') def test_authentication_required(self, method): """Test and verify that authentication is required for this endpoint.""" self.client.logout() @@ -1961,31 +1761,29 @@ def test_oauth(self): self.client.logout() response = self.client.get(self.path(), **oauth_headers) assert response.status_code == 200 - body = {"user_id": "staff", "action": "allow"} - response = self.client.post(self.path(), body, format="json", **oauth_headers) + body = {'user_id': 'staff', 'action': 'allow'} + response = self.client.post(self.path(), body, format='json', **oauth_headers) assert response.status_code == 200 @ddt.data( - {"username": "u1", "is_staff": False, "expected_status": 403}, - {"username": "u2", "is_staff": True, "expected_status": 200}, + {'username': 'u1', 'is_staff': False, 'expected_status': 403}, + {'username': 'u2', 'is_staff': True, 'expected_status': 200}, ) @ddt.unpack def test_staff_permission_required(self, username, is_staff, expected_status): """Test and verify that only users with staff permission can access this endpoint.""" - UserFactory(username=username, password="edx", is_staff=is_staff) - self.client.login(username=username, password="edx") + UserFactory(username=username, password='edx', is_staff=is_staff) + self.client.login(username=username, password='edx') response = self.client.get(self.path()) assert response.status_code == expected_status - response = self.client.post( - self.path(), {"user_id": username, "action": "allow"}, format="json" - ) + response = self.client.post(self.path(), {'user_id': username, 'action': 'allow'}, format='json') assert response.status_code == expected_status def test_non_existent_course_id(self): """Test the response when the endpoint URL contains a non-existent course id.""" self._login_as_staff() - path = self.path(course_id="course-v1:a+b+c") + path = self.path(course_id='course-v1:a+b+c') response = self.client.get(path) assert response.status_code == 404 @@ -1996,7 +1794,7 @@ def test_non_existent_course_id(self): def test_non_existent_course_role(self): """Test the response when the endpoint URL contains a non-existent role.""" self._login_as_staff() - path = self.path(role="A") + path = self.path(role='A') response = self.client.get(path) assert response.status_code == 400 @@ -2005,10 +1803,10 @@ def test_non_existent_course_role(self): assert response.status_code == 400 @ddt.data( - {"role": "Moderator", "count": 0}, - {"role": "Moderator", "count": 1}, - {"role": "Group Moderator", "count": 2}, - {"role": "Community TA", "count": 3}, + {'role': 'Moderator', 'count': 0}, + {'role': 'Moderator', 'count': 1}, + {'role': 'Group Moderator', 'count': 2}, + {'role': 'Community TA', 'count': 3}, ) @ddt.unpack def test_get_role_members(self, role, count): @@ -2022,14 +1820,14 @@ def test_get_role_members(self, role, count): assert response.status_code == 200 - content = json.loads(response.content.decode("utf-8")) - assert content["course_id"] == "course-v1:x+y+z" - assert len(content["results"]) == count - expected_fields = ("username", "email", "first_name", "last_name", "group_name") - for item in content["results"]: + content = json.loads(response.content.decode('utf-8')) + assert content['course_id'] == 'course-v1:x+y+z' + assert len(content['results']) == count + expected_fields = ('username', 'email', 'first_name', 'last_name', 'group_name') + for item in content['results']: for expected_field in expected_fields: assert expected_field in item - assert content["division_scheme"] == "cohort" + assert content['division_scheme'] == 'cohort' def test_post_missing_body(self): """Test the response with a POST request without a body.""" @@ -2038,9 +1836,9 @@ def test_post_missing_body(self): assert response.status_code == 400 @ddt.data( - {"a": 1}, - {"user_id": "xyz", "action": "allow"}, - {"user_id": "staff", "action": 123}, + {'a': 1}, + {'user_id': 'xyz', 'action': 'allow'}, + {'user_id': 'staff', 'action': 123}, ) def test_missing_or_invalid_parameters(self, body): """ @@ -2051,100 +1849,82 @@ def test_missing_or_invalid_parameters(self, body): response = self.client.post(self.path(), body) assert response.status_code == 400 - response = self.client.post(self.path(), body, format="json") + response = self.client.post(self.path(), body, format='json') assert response.status_code == 400 @ddt.data( - {"action": "allow", "user_in_role": False}, - {"action": "allow", "user_in_role": True}, - {"action": "revoke", "user_in_role": False}, - {"action": "revoke", "user_in_role": True}, + {'action': 'allow', 'user_in_role': False}, + {'action': 'allow', 'user_in_role': True}, + {'action': 'revoke', 'user_in_role': False}, + {'action': 'revoke', 'user_in_role': True} ) @ddt.unpack def test_post_update_user_role(self, action, user_in_role): """Test the response when updating the user's role""" users = self._create_and_enroll_users(count=1) user = users[0] - role = "Moderator" + role = 'Moderator' if user_in_role: self._add_users_to_role(users, role) response = self.post(role, user.username, action) assert response.status_code == 200 - content = json.loads(response.content.decode("utf-8")) - assertion = self.assertTrue if action == "allow" else self.assertFalse - assertion(any(user.username in x["username"] for x in content["results"])) + content = json.loads(response.content.decode('utf-8')) + assertion = self.assertTrue if action == 'allow' else self.assertFalse + assertion(any(user.username in x['username'] for x in content['results'])) @ddt.ddt @httpretty.activate @override_waffle_flag(ENABLE_DISCUSSIONS_MFE, True) -class CourseActivityStatsTest( - ForumsEnableMixin, - UrlResetMixin, - CommentsServiceMockMixin, - APITestCase, - SharedModuleStoreTestCase, -): +class CourseActivityStatsTest(ForumsEnableMixin, UrlResetMixin, CommentsServiceMockMixin, APITestCase, + SharedModuleStoreTestCase): """ Tests for the course stats endpoint """ - @mock.patch.dict( - "django.conf.settings.FEATURES", {"ENABLE_DISCUSSION_SERVICE": True} - ) + @mock.patch.dict("django.conf.settings.FEATURES", {"ENABLE_DISCUSSION_SERVICE": True}) def setUp(self) -> None: super().setUp() patcher = mock.patch( - "openedx.core.djangoapps.discussions.config.waffle.ENABLE_FORUM_V2.is_enabled", - return_value=False, + 'openedx.core.djangoapps.discussions.config.waffle.ENABLE_FORUM_V2.is_enabled', + return_value=False ) patcher.start() self.addCleanup(patcher.stop) self.course = CourseFactory.create() self.course_key = str(self.course.id) seed_permissions_roles(self.course.id) - self.user = UserFactory(username="user") - self.moderator = UserFactory(username="moderator") + self.user = UserFactory(username='user') + self.moderator = UserFactory(username='moderator') moderator_role = Role.objects.get(name="Moderator", course_id=self.course.id) moderator_role.users.add(self.moderator) self.stats = [ { - "threads": random.randint(0, 10), - "replies": random.randint(0, 30), - "responses": random.randint(0, 100), - "deleted_threads": 0, - "deleted_replies": 0, - "deleted_responses": 0, "active_flags": random.randint(0, 3), "inactive_flags": random.randint(0, 2), - "username": f"user-{idx}", + "replies": random.randint(0, 30), + "responses": random.randint(0, 100), + "threads": random.randint(0, 10), + "username": f"user-{idx}" } for idx in range(10) ] for stat in self.stats: user = UserFactory.create( - username=stat["username"], + username=stat['username'], email=f"{stat['username']}@example.com", - password=self.TEST_PASSWORD, + password=self.TEST_PASSWORD ) - CourseEnrollment.enroll(user, self.course.id, mode="audit") + CourseEnrollment.enroll(user, self.course.id, mode='audit') - CourseEnrollment.enroll(self.moderator, self.course.id, mode="audit") - self.stats_without_flags = [ - {**stat, "active_flags": None, "inactive_flags": None} - for stat in self.stats - ] + CourseEnrollment.enroll(self.moderator, self.course.id, mode='audit') + self.stats_without_flags = [{**stat, "active_flags": None, "inactive_flags": None} for stat in self.stats] self.register_course_stats_response(self.course_key, self.stats, 1, 3) - self.url = reverse( - "discussion_course_activity_stats", - kwargs={"course_key_string": self.course_key}, - ) + self.url = reverse("discussion_course_activity_stats", kwargs={"course_key_string": self.course_key}) - @mock.patch.dict( - "django.conf.settings.FEATURES", {"ENABLE_DISCUSSION_SERVICE": True} - ) + @mock.patch.dict("django.conf.settings.FEATURES", {"ENABLE_DISCUSSION_SERVICE": True}) def test_regular_user(self): """ Tests that for a regular user stats are returned without flag counts @@ -2154,9 +1934,7 @@ def test_regular_user(self): data = response.json() assert data["results"] == self.stats_without_flags - @mock.patch.dict( - "django.conf.settings.FEATURES", {"ENABLE_DISCUSSION_SERVICE": True} - ) + @mock.patch.dict("django.conf.settings.FEATURES", {"ENABLE_DISCUSSION_SERVICE": True}) def test_moderator_user(self): """ Tests that for a moderator user stats are returned with flag counts @@ -2176,9 +1954,7 @@ def test_moderator_user(self): ("user", "recency", "recency"), ) @ddt.unpack - @mock.patch.dict( - "django.conf.settings.FEATURES", {"ENABLE_DISCUSSION_SERVICE": True} - ) + @mock.patch.dict("django.conf.settings.FEATURES", {"ENABLE_DISCUSSION_SERVICE": True}) def test_sorting(self, username, ordering_requested, ordering_performed): """ Test valid sorting options and defaults @@ -2188,22 +1964,15 @@ def test_sorting(self, username, ordering_requested, ordering_performed): if ordering_requested: params = {"order_by": ordering_requested} self.client.get(self.url, params) - assert ( - urlparse( - httpretty.last_request().path # lint-amnesty, pylint: disable=no-member - ).path - == f"/api/v1/users/{self.course_key}/stats" - ) + assert urlparse( + httpretty.last_request().path # lint-amnesty, pylint: disable=no-member + ).path == f"/api/v1/users/{self.course_key}/stats" assert parse_qs( - urlparse( - httpretty.last_request().path - ).query # lint-amnesty, pylint: disable=no-member + urlparse(httpretty.last_request().path).query # lint-amnesty, pylint: disable=no-member ).get("sort_key", None) == [ordering_performed] @ddt.data("flagged", "xyz") - @mock.patch.dict( - "django.conf.settings.FEATURES", {"ENABLE_DISCUSSION_SERVICE": True} - ) + @mock.patch.dict("django.conf.settings.FEATURES", {"ENABLE_DISCUSSION_SERVICE": True}) def test_sorting_error_regular_user(self, order_by): """ Test for invalid sorting options for regular users. @@ -2213,60 +1982,47 @@ def test_sorting_error_regular_user(self, order_by): assert "order_by" in response.json()["field_errors"] @ddt.data( - ( - "user", - "user-0,user-1,user-2,user-3,user-4,user-5,user-6,user-7,user-8,user-9", - ), - ("moderator", "moderator"), + ('user', 'user-0,user-1,user-2,user-3,user-4,user-5,user-6,user-7,user-8,user-9'), + ('moderator', 'moderator'), ) @ddt.unpack - @mock.patch.dict( - "django.conf.settings.FEATURES", {"ENABLE_DISCUSSION_SERVICE": True} - ) - def test_with_username_param( - self, username_search_string, comma_separated_usernames - ): + @mock.patch.dict("django.conf.settings.FEATURES", {'ENABLE_DISCUSSION_SERVICE': True}) + def test_with_username_param(self, username_search_string, comma_separated_usernames): """ Test for endpoint with username param. """ - params = {"username": username_search_string} + params = {'username': username_search_string} self.client.login(username=self.moderator.username, password=self.TEST_PASSWORD) self.client.get(self.url, params) - assert ( - urlparse( - httpretty.last_request().path # lint-amnesty, pylint: disable=no-member - ).path - == f"/api/v1/users/{self.course_key}/stats" - ) + assert urlparse( + httpretty.last_request().path # lint-amnesty, pylint: disable=no-member + ).path == f'/api/v1/users/{self.course_key}/stats' assert parse_qs( - urlparse( - httpretty.last_request().path - ).query # lint-amnesty, pylint: disable=no-member - ).get("usernames", [None]) == [comma_separated_usernames] + urlparse(httpretty.last_request().path).query # lint-amnesty, pylint: disable=no-member + ).get('usernames', [None]) == [comma_separated_usernames] - @mock.patch.dict( - "django.conf.settings.FEATURES", {"ENABLE_DISCUSSION_SERVICE": True} - ) + @mock.patch.dict("django.conf.settings.FEATURES", {'ENABLE_DISCUSSION_SERVICE': True}) def test_with_username_param_with_no_matches(self): """ Test for endpoint with username param with no matches. """ - params = {"username": "unknown"} + params = {'username': 'unknown'} self.client.login(username=self.moderator.username, password=self.TEST_PASSWORD) response = self.client.get(self.url, params) data = response.json() - self.assertFalse(data["results"]) - assert data["pagination"]["count"] == 0 + self.assertFalse(data['results']) + assert data['pagination']['count'] == 0 - @ddt.data("user-0", "USER-1", "User-2", "UsEr-3") - @mock.patch.dict( - "django.conf.settings.FEATURES", {"ENABLE_DISCUSSION_SERVICE": True} + @ddt.data( + 'user-0', + 'USER-1', + 'User-2', + 'UsEr-3' ) + @mock.patch.dict("django.conf.settings.FEATURES", {'ENABLE_DISCUSSION_SERVICE': True}) def test_with_username_param_case(self, username_search_string): """ Test user search function is case-insensitive. """ - response = get_usernames_from_search_string( - self.course_key, username_search_string, 1, 1 - ) + response = get_usernames_from_search_string(self.course_key, username_search_string, 1, 1) assert response == (username_search_string.lower(), 1, 1) diff --git a/lms/djangoapps/discussion/rest_api/tests/test_views_v2.py b/lms/djangoapps/discussion/rest_api/tests/test_views_v2.py index 39e48dd41ff9..431304a9a2b5 100644 --- a/lms/djangoapps/discussion/rest_api/tests/test_views_v2.py +++ b/lms/djangoapps/discussion/rest_api/tests/test_views_v2.py @@ -14,6 +14,8 @@ from unittest import mock import ddt +from forum.backends.mongodb.comments import Comment +from forum.backends.mongodb.threads import CommentThread import httpretty from django.urls import reverse from pytz import UTC @@ -21,39 +23,30 @@ from rest_framework.parsers import JSONParser from rest_framework.test import APIClient +from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase +from xmodule.modulestore.tests.factories import CourseFactory from common.djangoapps.student.tests.factories import ( CourseEnrollmentFactory, UserFactory, ) from common.djangoapps.util.testing import PatchMediaTypeMixin, UrlResetMixin from common.test.utils import disable_signal -from forum.backends.mongodb.comments import Comment -from forum.backends.mongodb.threads import CommentThread -from lms.djangoapps.discussion.django_comment_client.tests.utils import ( - ForumsEnableMixin, +from lms.djangoapps.discussion.tests.utils import ( + make_minimal_cs_comment, + make_minimal_cs_thread, ) +from lms.djangoapps.discussion.django_comment_client.tests.utils import ForumsEnableMixin from lms.djangoapps.discussion.rest_api import api from lms.djangoapps.discussion.rest_api.tests.utils import ( ForumMockUtilsMixin, ProfileImageTestMixin, make_paginated_api_response, ) -from lms.djangoapps.discussion.tests.utils import ( - make_minimal_cs_comment, - make_minimal_cs_thread, -) from openedx.core.djangoapps.django_comment_common.models import ( - FORUM_ROLE_ADMINISTRATOR, - FORUM_ROLE_COMMUNITY_TA, - FORUM_ROLE_MODERATOR, - FORUM_ROLE_STUDENT, - assign_role, + FORUM_ROLE_ADMINISTRATOR, FORUM_ROLE_COMMUNITY_TA, FORUM_ROLE_MODERATOR, FORUM_ROLE_STUDENT, + assign_role ) -from openedx.core.djangoapps.user_api.accounts.image_helpers import ( - get_profile_image_storage, -) -from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase -from xmodule.modulestore.tests.factories import CourseFactory +from openedx.core.djangoapps.user_api.accounts.image_helpers import get_profile_image_storage class DiscussionAPIViewTestMixin(ForumsEnableMixin, ForumMockUtilsMixin, UrlResetMixin): @@ -394,10 +387,6 @@ def expected_response_data(self, overrides=None): "image_url_small": "http://testserver/static/default_30.png", }, "learner_status": "new", - "is_deleted": None, - "deleted_at": None, - "deleted_by": None, - "deleted_by_label": None, } response_data.update(overrides or {}) return response_data @@ -523,17 +512,15 @@ def test_course_id_missing(self): self.assert_response_correct( response, 400, - { - "field_errors": { - "course_id": {"developer_message": "This field is required."} - } - }, + {"field_errors": {"course_id": {"developer_message": "This field is required."}}} ) def test_404(self): response = self.client.get(self.url, {"course_id": "non/existent/course"}) self.assert_response_correct( - response, 404, {"developer_message": "Course not found."} + response, + 404, + {"developer_message": "Course not found."} ) def test_basic(self): @@ -884,9 +871,7 @@ class BulkDeleteUserPostsTest(DiscussionAPIViewTestMixin, ModuleStoreTestCase): def setUp(self): super().setUp() - self.url = reverse( - "bulk_delete_user_posts", kwargs={"course_id": str(self.course.id)} - ) + self.url = reverse("bulk_delete_user_posts", kwargs={"course_id": str(self.course.id)}) self.user2 = UserFactory.create(password=self.password) CourseEnrollmentFactory.create(user=self.user2, course_id=self.course.id) @@ -902,19 +887,13 @@ def mock_comment_and_thread_count(self, comment_count=1, thread_count=1): thread_collection = mock.MagicMock() thread_collection.count_documents.return_value = thread_count patch_thread = mock.patch.object( - CommentThread, - "_collection", - new_callable=mock.PropertyMock, - return_value=thread_collection, + CommentThread, "_collection", new_callable=mock.PropertyMock, return_value=thread_collection ) comment_collection = mock.MagicMock() comment_collection.count_documents.return_value = comment_count patch_comment = mock.patch.object( - Comment, - "_collection", - new_callable=mock.PropertyMock, - return_value=comment_collection, + Comment, "_collection", new_callable=mock.PropertyMock, return_value=comment_collection ) thread_mock = patch_thread.start() @@ -929,9 +908,7 @@ def test_bulk_delete_denied_for_discussion_roles(self, role): """ Test bulk delete user posts denied with discussion roles. """ - thread_mock, comment_mock = self.mock_comment_and_thread_count( - comment_count=1, thread_count=1 - ) + thread_mock, comment_mock = self.mock_comment_and_thread_count(comment_count=1, thread_count=1) assign_role(self.course.id, self.user, role) response = self.client.post( f"{self.url}?username={self.user2.username}", @@ -955,9 +932,7 @@ def test_bulk_delete_allowed_for_discussion_roles(self, role): assert response.status_code == status.HTTP_202_ACCEPTED assert response.json() == {"comment_count": 1, "thread_count": 1} - @mock.patch( - "lms.djangoapps.discussion.rest_api.views.delete_course_post_for_user.apply_async" - ) + @mock.patch('lms.djangoapps.discussion.rest_api.views.delete_course_post_for_user.apply_async') @ddt.data(True, False) def test_task_only_runs_if_execute_param_is_true(self, execute, task_mock): """ diff --git a/lms/djangoapps/discussion/rest_api/tests/utils.py b/lms/djangoapps/discussion/rest_api/tests/utils.py index 990e68a30af9..8c1615690ad5 100644 --- a/lms/djangoapps/discussion/rest_api/tests/utils.py +++ b/lms/djangoapps/discussion/rest_api/tests/utils.py @@ -2,6 +2,7 @@ Discussion API test utilities """ + import hashlib import json import re @@ -13,18 +14,11 @@ from PIL import Image from pytz import UTC -from lms.djangoapps.discussion.django_comment_client.tests.mixins import ( - MockForumApiMixin, -) -from openedx.core.djangoapps.django_comment_common.comment_client.utils import ( - CommentClientRequestError, -) +from lms.djangoapps.discussion.django_comment_client.tests.mixins import MockForumApiMixin +from openedx.core.djangoapps.django_comment_common.comment_client.utils import CommentClientRequestError from openedx.core.djangoapps.profile_images.images import create_profile_images from openedx.core.djangoapps.profile_images.tests.helpers import make_image_file -from openedx.core.djangoapps.user_api.accounts.image_helpers import ( - get_profile_image_names, - set_has_profile_image, -) +from openedx.core.djangoapps.user_api.accounts.image_helpers import get_profile_image_names, set_has_profile_image def _get_thread_callback(thread_data): @@ -32,7 +26,6 @@ def _get_thread_callback(thread_data): Get a callback function that will return POST/PUT data overridden by response_overrides. """ - def callback(request, _uri, headers): """ Simulate the thread creation or update endpoint by returning the provided @@ -49,7 +42,7 @@ def callback(request, _uri, headers): response_data["edit_history"] = [ { "original_body": original_data["body"], - "author": thread_data.get("username"), + "author": thread_data.get('username'), "reason_code": val, }, ] @@ -75,13 +68,11 @@ def callback(*args, **kwargs): if key in ["anonymous", "anonymous_to_peers", "closed", "pinned"]: response_data[key] = val is True or val == "True" elif key == "edit_reason_code": - response_data["edit_history"] = [ - { - "original_body": original_data["body"], - "author": thread_data.get("username"), - "reason_code": val, - } - ] + response_data["edit_history"] = [{ + "original_body": original_data["body"], + "author": thread_data.get("username"), + "reason_code": val, + }] else: response_data[key] = val @@ -96,7 +87,6 @@ def _get_comment_callback(comment_data, thread_id, parent_id): plus necessary dummy data, overridden by the content of the POST/PUT request. """ - def callback(request, _uri, headers): """ Simulate the comment creation or update endpoint as described above. @@ -115,7 +105,7 @@ def callback(request, _uri, headers): response_data["edit_history"] = [ { "original_body": original_data["body"], - "author": comment_data.get("username"), + "author": comment_data.get('username'), "reason_code": val, }, ] @@ -145,13 +135,11 @@ def callback(*args, **kwargs): if key in ["anonymous", "anonymous_to_peers", "endorsed"]: response_data[key] = val is True or val == "True" elif key == "edit_reason_code": - response_data["edit_history"] = [ - { - "original_body": original_data["body"], - "author": comment_data.get("username"), - "reason_code": val, - } - ] + response_data["edit_history"] = [{ + "original_body": original_data["body"], + "author": comment_data.get("username"), + "reason_code": val, + }] else: response_data[key] = val @@ -164,11 +152,9 @@ def make_user_callbacks(user_map): """ Returns a callable that mimics user creation. """ - def callback(*args, **kwargs): - user_id = args[0] if args else kwargs.get("user_id") + user_id = args[0] if args else kwargs.get('user_id') return user_map[str(user_id)] - return callback @@ -177,58 +163,54 @@ class CommentsServiceMockMixin: def register_get_threads_response(self, threads, page, num_pages): """Register a mock response for GET on the CS thread list endpoint""" - assert httpretty.is_enabled(), "httpretty must be enabled to mock calls." + assert httpretty.is_enabled(), 'httpretty must be enabled to mock calls.' httpretty.register_uri( httpretty.GET, "http://localhost:4567/api/v1/threads", - body=json.dumps( - { - "collection": threads, - "page": page, - "num_pages": num_pages, - "thread_count": len(threads), - } - ), - status=200, + body=json.dumps({ + "collection": threads, + "page": page, + "num_pages": num_pages, + "thread_count": len(threads), + }), + status=200 ) def register_get_course_commentable_counts_response(self, course_id, thread_counts): """Register a mock response for GET on the CS thread list endpoint""" - assert httpretty.is_enabled(), "httpretty must be enabled to mock calls." + assert httpretty.is_enabled(), 'httpretty must be enabled to mock calls.' httpretty.register_uri( httpretty.GET, f"http://localhost:4567/api/v1/commentables/{course_id}/counts", body=json.dumps(thread_counts), - status=200, + status=200 ) def register_get_threads_search_response(self, threads, rewrite, num_pages=1): """Register a mock response for GET on the CS thread search endpoint""" - assert httpretty.is_enabled(), "httpretty must be enabled to mock calls." + assert httpretty.is_enabled(), 'httpretty must be enabled to mock calls.' httpretty.register_uri( httpretty.GET, "http://localhost:4567/api/v1/search/threads", - body=json.dumps( - { - "collection": threads, - "page": 1, - "num_pages": num_pages, - "corrected_text": rewrite, - "thread_count": len(threads), - } - ), - status=200, + body=json.dumps({ + "collection": threads, + "page": 1, + "num_pages": num_pages, + "corrected_text": rewrite, + "thread_count": len(threads), + }), + status=200 ) def register_post_thread_response(self, thread_data): """Register a mock response for POST on the CS commentable endpoint""" - assert httpretty.is_enabled(), "httpretty must be enabled to mock calls." + assert httpretty.is_enabled(), 'httpretty must be enabled to mock calls.' httpretty.register_uri( httpretty.POST, re.compile(r"http://localhost:4567/api/v1/(\w+)/threads"), - body=_get_thread_callback(thread_data), + body=_get_thread_callback(thread_data) ) def register_put_thread_response(self, thread_data): @@ -236,51 +218,49 @@ def register_put_thread_response(self, thread_data): Register a mock response for PUT on the CS endpoint for the given thread_id. """ - assert httpretty.is_enabled(), "httpretty must be enabled to mock calls." + assert httpretty.is_enabled(), 'httpretty must be enabled to mock calls.' httpretty.register_uri( httpretty.PUT, "http://localhost:4567/api/v1/threads/{}".format(thread_data["id"]), - body=_get_thread_callback(thread_data), + body=_get_thread_callback(thread_data) ) def register_get_thread_error_response(self, thread_id, status_code): """Register a mock error response for GET on the CS thread endpoint.""" - assert httpretty.is_enabled(), "httpretty must be enabled to mock calls." + assert httpretty.is_enabled(), 'httpretty must be enabled to mock calls.' httpretty.register_uri( httpretty.GET, f"http://localhost:4567/api/v1/threads/{thread_id}", body="", - status=status_code, + status=status_code ) def register_get_thread_response(self, thread): """ Register a mock response for GET on the CS thread instance endpoint. """ - assert httpretty.is_enabled(), "httpretty must be enabled to mock calls." + assert httpretty.is_enabled(), 'httpretty must be enabled to mock calls.' httpretty.register_uri( httpretty.GET, "http://localhost:4567/api/v1/threads/{id}".format(id=thread["id"]), body=json.dumps(thread), - status=200, + status=200 ) def register_get_comments_response(self, comments, page, num_pages): """Register a mock response for GET on the CS comments list endpoint""" - assert httpretty.is_enabled(), "httpretty must be enabled to mock calls." + assert httpretty.is_enabled(), 'httpretty must be enabled to mock calls.' httpretty.register_uri( httpretty.GET, "http://localhost:4567/api/v1/comments", - body=json.dumps( - { - "collection": comments, - "page": page, - "num_pages": num_pages, - "comment_count": len(comments), - } - ), - status=200, + body=json.dumps({ + "collection": comments, + "page": page, + "num_pages": num_pages, + "comment_count": len(comments), + }), + status=200 ) def register_post_comment_response(self, comment_data, thread_id, parent_id=None): @@ -294,11 +274,11 @@ def register_post_comment_response(self, comment_data, thread_id, parent_id=None else: url = f"http://localhost:4567/api/v1/threads/{thread_id}/comments" - assert httpretty.is_enabled(), "httpretty must be enabled to mock calls." + assert httpretty.is_enabled(), 'httpretty must be enabled to mock calls.' httpretty.register_uri( httpretty.POST, url, - body=_get_comment_callback(comment_data, thread_id, parent_id), + body=_get_comment_callback(comment_data, thread_id, parent_id) ) def register_put_comment_response(self, comment_data): @@ -308,11 +288,11 @@ def register_put_comment_response(self, comment_data): """ thread_id = comment_data["thread_id"] parent_id = comment_data.get("parent_id") - assert httpretty.is_enabled(), "httpretty must be enabled to mock calls." + assert httpretty.is_enabled(), 'httpretty must be enabled to mock calls.' httpretty.register_uri( httpretty.PUT, "http://localhost:4567/api/v1/comments/{}".format(comment_data["id"]), - body=_get_comment_callback(comment_data, thread_id, parent_id), + body=_get_comment_callback(comment_data, thread_id, parent_id) ) def register_get_comment_error_response(self, comment_id, status_code): @@ -320,12 +300,12 @@ def register_get_comment_error_response(self, comment_id, status_code): Register a mock error response for GET on the CS comment instance endpoint. """ - assert httpretty.is_enabled(), "httpretty must be enabled to mock calls." + assert httpretty.is_enabled(), 'httpretty must be enabled to mock calls.' httpretty.register_uri( httpretty.GET, f"http://localhost:4567/api/v1/comments/{comment_id}", body="", - status=status_code, + status=status_code ) def register_get_comment_response(self, response_overrides): @@ -333,83 +313,75 @@ def register_get_comment_response(self, response_overrides): Register a mock response for GET on the CS comment instance endpoint. """ comment = make_minimal_cs_comment(response_overrides) - assert httpretty.is_enabled(), "httpretty must be enabled to mock calls." + assert httpretty.is_enabled(), 'httpretty must be enabled to mock calls.' httpretty.register_uri( httpretty.GET, "http://localhost:4567/api/v1/comments/{id}".format(id=comment["id"]), body=json.dumps(comment), - status=200, + status=200 ) - def register_get_user_response( - self, user, subscribed_thread_ids=None, upvoted_ids=None - ): + def register_get_user_response(self, user, subscribed_thread_ids=None, upvoted_ids=None): """Register a mock response for GET on the CS user instance endpoint""" - assert httpretty.is_enabled(), "httpretty must be enabled to mock calls." + assert httpretty.is_enabled(), 'httpretty must be enabled to mock calls.' httpretty.register_uri( httpretty.GET, f"http://localhost:4567/api/v1/users/{user.id}", - body=json.dumps( - { - "id": str(user.id), - "subscribed_thread_ids": subscribed_thread_ids or [], - "upvoted_ids": upvoted_ids or [], - } - ), - status=200, + body=json.dumps({ + "id": str(user.id), + "subscribed_thread_ids": subscribed_thread_ids or [], + "upvoted_ids": upvoted_ids or [], + }), + status=200 ) def register_get_user_retire_response(self, user, status=200, body=""): """Register a mock response for GET on the CS user retirement endpoint""" - assert httpretty.is_enabled(), "httpretty must be enabled to mock calls." + assert httpretty.is_enabled(), 'httpretty must be enabled to mock calls.' httpretty.register_uri( httpretty.POST, f"http://localhost:4567/api/v1/users/{user.id}/retire", body=body, - status=status, + status=status ) def register_get_username_replacement_response(self, user, status=200, body=""): - assert httpretty.is_enabled(), "httpretty must be enabled to mock calls." + assert httpretty.is_enabled(), 'httpretty must be enabled to mock calls.' httpretty.register_uri( httpretty.POST, f"http://localhost:4567/api/v1/users/{user.id}/replace_username", body=body, - status=status, + status=status ) def register_subscribed_threads_response(self, user, threads, page, num_pages): """Register a mock response for GET on the CS user instance endpoint""" - assert httpretty.is_enabled(), "httpretty must be enabled to mock calls." + assert httpretty.is_enabled(), 'httpretty must be enabled to mock calls.' httpretty.register_uri( httpretty.GET, f"http://localhost:4567/api/v1/users/{user.id}/subscribed_threads", - body=json.dumps( - { - "collection": threads, - "page": page, - "num_pages": num_pages, - "thread_count": len(threads), - } - ), - status=200, + body=json.dumps({ + "collection": threads, + "page": page, + "num_pages": num_pages, + "thread_count": len(threads), + }), + status=200 ) def register_course_stats_response(self, course_key, stats, page, num_pages): """Register a mock response for GET on the CS user course stats instance endpoint""" - assert httpretty.is_enabled(), "httpretty must be enabled to mock calls." + assert httpretty.is_enabled(), 'httpretty must be enabled to mock calls.' httpretty.register_uri( httpretty.GET, f"http://localhost:4567/api/v1/users/{course_key}/stats", - body=json.dumps( - { - "user_stats": stats, - "page": page, - "num_pages": num_pages, - "count": len(stats), - } - ), - status=200, + body=json.dumps({ + "user_stats": stats, + "page": page, + "num_pages": num_pages, + "count": len(stats), + }), + status=200 ) def register_subscription_response(self, user): @@ -417,13 +389,13 @@ def register_subscription_response(self, user): Register a mock response for POST and DELETE on the CS user subscription endpoint """ - assert httpretty.is_enabled(), "httpretty must be enabled to mock calls." + assert httpretty.is_enabled(), 'httpretty must be enabled to mock calls.' for method in [httpretty.POST, httpretty.DELETE]: httpretty.register_uri( method, f"http://localhost:4567/api/v1/users/{user.id}/subscriptions", body=json.dumps({}), # body is unused - status=200, + status=200 ) def register_thread_votes_response(self, thread_id): @@ -431,13 +403,13 @@ def register_thread_votes_response(self, thread_id): Register a mock response for PUT and DELETE on the CS thread votes endpoint """ - assert httpretty.is_enabled(), "httpretty must be enabled to mock calls." + assert httpretty.is_enabled(), 'httpretty must be enabled to mock calls.' for method in [httpretty.PUT, httpretty.DELETE]: httpretty.register_uri( method, f"http://localhost:4567/api/v1/threads/{thread_id}/votes", body=json.dumps({}), # body is unused - status=200, + status=200 ) def register_comment_votes_response(self, comment_id): @@ -445,39 +417,41 @@ def register_comment_votes_response(self, comment_id): Register a mock response for PUT and DELETE on the CS comment votes endpoint """ - assert httpretty.is_enabled(), "httpretty must be enabled to mock calls." + assert httpretty.is_enabled(), 'httpretty must be enabled to mock calls.' for method in [httpretty.PUT, httpretty.DELETE]: httpretty.register_uri( method, f"http://localhost:4567/api/v1/comments/{comment_id}/votes", body=json.dumps({}), # body is unused - status=200, + status=200 ) def register_flag_response(self, content_type, content_id): """Register a mock response for PUT on the CS flag endpoints""" - assert httpretty.is_enabled(), "httpretty must be enabled to mock calls." + assert httpretty.is_enabled(), 'httpretty must be enabled to mock calls.' for path in ["abuse_flag", "abuse_unflag"]: httpretty.register_uri( "PUT", "http://localhost:4567/api/v1/{content_type}s/{content_id}/{path}".format( - content_type=content_type, content_id=content_id, path=path + content_type=content_type, + content_id=content_id, + path=path ), body=json.dumps({}), # body is unused - status=200, + status=200 ) def register_read_response(self, user, content_type, content_id): """ Register a mock response for POST on the CS 'read' endpoint """ - assert httpretty.is_enabled(), "httpretty must be enabled to mock calls." + assert httpretty.is_enabled(), 'httpretty must be enabled to mock calls.' httpretty.register_uri( httpretty.POST, f"http://localhost:4567/api/v1/users/{user.id}/read", - params={"source_type": content_type, "source_id": content_id}, + params={'source_type': content_type, 'source_id': content_id}, body=json.dumps({}), # body is unused - status=200, + status=200 ) def register_thread_flag_response(self, thread_id): @@ -492,48 +466,48 @@ def register_delete_thread_response(self, thread_id): """ Register a mock response for DELETE on the CS thread instance endpoint """ - assert httpretty.is_enabled(), "httpretty must be enabled to mock calls." + assert httpretty.is_enabled(), 'httpretty must be enabled to mock calls.' httpretty.register_uri( httpretty.DELETE, f"http://localhost:4567/api/v1/threads/{thread_id}", body=json.dumps({}), # body is unused - status=200, + status=200 ) def register_delete_comment_response(self, comment_id): """ Register a mock response for DELETE on the CS comment instance endpoint """ - assert httpretty.is_enabled(), "httpretty must be enabled to mock calls." + assert httpretty.is_enabled(), 'httpretty must be enabled to mock calls.' httpretty.register_uri( httpretty.DELETE, f"http://localhost:4567/api/v1/comments/{comment_id}", body=json.dumps({}), # body is unused - status=200, + status=200 ) def register_user_active_threads(self, user_id, response): """ Register a mock response for GET on the CS comment active threads endpoint """ - assert httpretty.is_enabled(), "httpretty must be enabled to mock calls." + assert httpretty.is_enabled(), 'httpretty must be enabled to mock calls.' httpretty.register_uri( httpretty.GET, f"http://localhost:4567/api/v1/users/{user_id}/active_threads", body=json.dumps(response), - status=200, + status=200 ) def register_get_subscriptions(self, thread_id, response): """ Register a mock response for GET on the CS comment active threads endpoint """ - assert httpretty.is_enabled(), "httpretty must be enabled to mock calls." + assert httpretty.is_enabled(), 'httpretty must be enabled to mock calls.' httpretty.register_uri( httpretty.GET, f"http://localhost:4567/api/v1/threads/{thread_id}/subscriptions", body=json.dumps(response), - status=200, + status=200 ) def assert_query_params_equal(self, httpretty_request, expected_params): @@ -557,7 +531,7 @@ def request_patch(self, request_data): return self.client.patch( self.url, json.dumps(request_data), - content_type="application/merge-patch+json", + content_type="application/merge-patch+json" ) def expected_thread_data(self, overrides=None): @@ -615,10 +589,6 @@ def expected_thread_data(self, overrides=None): "close_reason": None, "close_reason_code": None, "learner_status": "new", - "is_deleted": None, - "deleted_at": None, - "deleted_by": None, - "deleted_by_label": None, } response_data.update(overrides or {}) return response_data @@ -629,153 +599,137 @@ class ForumMockUtilsMixin(MockForumApiMixin): def register_get_threads_response(self, threads, page, num_pages): """Register a mock response for GET on the CS thread list endpoint""" - self.set_mock_return_value( - "get_user_threads", - { - "collection": threads, - "page": page, - "num_pages": num_pages, - "thread_count": len(threads), - }, - ) + self.set_mock_return_value('get_user_threads', { + "collection": threads, + "page": page, + "num_pages": num_pages, + "thread_count": len(threads), + }) def register_get_course_commentable_counts_response(self, course_id, thread_counts): """Register a mock response for GET on the CS thread list endpoint""" - self.set_mock_return_value("get_commentables_stats", thread_counts) + self.set_mock_return_value('get_commentables_stats', thread_counts) def register_get_threads_search_response(self, threads, rewrite, num_pages=1): """Register a mock response for GET on the CS thread search endpoint""" - self.set_mock_return_value( - "search_threads", - { - "collection": threads, - "page": 1, - "num_pages": num_pages, - "corrected_text": rewrite, - "thread_count": len(threads), - }, - ) + self.set_mock_return_value('search_threads', { + "collection": threads, + "page": 1, + "num_pages": num_pages, + "corrected_text": rewrite, + "thread_count": len(threads), + }) def register_post_thread_response(self, thread_data): - self.set_mock_side_effect("create_thread", make_thread_callback(thread_data)) + self.set_mock_side_effect('create_thread', make_thread_callback(thread_data)) def register_put_thread_response(self, thread_data): - self.set_mock_side_effect("update_thread", make_thread_callback(thread_data)) + self.set_mock_side_effect('update_thread', make_thread_callback(thread_data)) def register_get_thread_error_response(self, thread_id, status_code): self.set_mock_side_effect( - "get_thread", - CommentClientRequestError(f"Thread does not exist with Id: {thread_id}"), + 'get_thread', + CommentClientRequestError(f"Thread does not exist with Id: {thread_id}") ) def register_get_thread_response(self, thread): - self.set_mock_return_value("get_thread", thread) + self.set_mock_return_value('get_thread', thread) def register_get_comments_response(self, comments, page, num_pages): - self.set_mock_return_value( - "get_parent_comment", - { - "collection": comments, - "page": page, - "num_pages": num_pages, - "comment_count": len(comments), - }, - ) + self.set_mock_return_value('get_parent_comment', { + "collection": comments, + "page": page, + "num_pages": num_pages, + "comment_count": len(comments), + }) def register_post_comment_response(self, comment_data, thread_id, parent_id=None): self.set_mock_side_effect( - "create_child_comment" if parent_id else "create_parent_comment", - make_comment_callback(comment_data, thread_id, parent_id), + 'create_child_comment' if parent_id else 'create_parent_comment', + make_comment_callback(comment_data, thread_id, parent_id) ) def register_put_comment_response(self, comment_data): thread_id = comment_data["thread_id"] parent_id = comment_data.get("parent_id") self.set_mock_side_effect( - "update_comment", make_comment_callback(comment_data, thread_id, parent_id) + 'update_comment', + make_comment_callback(comment_data, thread_id, parent_id) ) def register_get_comment_error_response(self, comment_id, status_code): self.set_mock_side_effect( - "get_parent_comment", - CommentClientRequestError(f"Comment does not exist with Id: {comment_id}"), + 'get_parent_comment', + CommentClientRequestError(f"Comment does not exist with Id: {comment_id}") ) def register_get_comment_response(self, response_overrides): comment = make_minimal_cs_comment(response_overrides) - self.set_mock_return_value("get_parent_comment", comment) + self.set_mock_return_value('get_parent_comment', comment) - def register_get_user_response( - self, user, subscribed_thread_ids=None, upvoted_ids=None - ): + def register_get_user_response(self, user, subscribed_thread_ids=None, upvoted_ids=None): """Register a mock response for GET on the CS user endpoint""" self.users_map[str(user.id)] = { "id": str(user.id), "subscribed_thread_ids": subscribed_thread_ids or [], "upvoted_ids": upvoted_ids or [], } - self.set_mock_side_effect("get_user", make_user_callbacks(self.users_map)) + self.set_mock_side_effect('get_user', make_user_callbacks(self.users_map)) def register_get_user_retire_response(self, user, body=""): - self.set_mock_return_value("retire_user", body) + self.set_mock_return_value('retire_user', body) def register_get_username_replacement_response(self, user, status=200, body=""): - self.set_mock_return_value("update_username", body) + self.set_mock_return_value('update_username', body) def register_subscribed_threads_response(self, user, threads, page, num_pages): - self.set_mock_return_value( - "get_user_subscriptions", - { - "collection": threads, - "page": page, - "num_pages": num_pages, - "thread_count": len(threads), - }, - ) + self.set_mock_return_value('get_user_subscriptions', { + "collection": threads, + "page": page, + "num_pages": num_pages, + "thread_count": len(threads), + }) def register_course_stats_response(self, course_key, stats, page, num_pages): - self.set_mock_return_value( - "get_user_course_stats", - { - "user_stats": stats, - "page": page, - "num_pages": num_pages, - "count": len(stats), - }, - ) + self.set_mock_return_value('get_user_course_stats', { + "user_stats": stats, + "page": page, + "num_pages": num_pages, + "count": len(stats), + }) def register_subscription_response(self, user): - self.set_mock_return_value("create_subscription", {}) - self.set_mock_return_value("delete_subscription", {}) + self.set_mock_return_value('create_subscription', {}) + self.set_mock_return_value('delete_subscription', {}) def register_thread_votes_response(self, thread_id): - self.set_mock_return_value("update_thread_votes", {}) - self.set_mock_return_value("delete_thread_vote", {}) + self.set_mock_return_value('update_thread_votes', {}) + self.set_mock_return_value('delete_thread_vote', {}) def register_comment_votes_response(self, comment_id): - self.set_mock_return_value("update_comment_votes", {}) - self.set_mock_return_value("delete_comment_vote", {}) + self.set_mock_return_value('update_comment_votes', {}) + self.set_mock_return_value('delete_comment_vote', {}) def register_flag_response(self, content_type, content_id): - if content_type == "thread": - self.set_mock_return_value("update_thread_flag", {}) - elif content_type == "comment": - self.set_mock_return_value("update_comment_flag", {}) + if content_type == 'thread': + self.set_mock_return_value('update_thread_flag', {}) + elif content_type == 'comment': + self.set_mock_return_value('update_comment_flag', {}) def register_read_response(self, user, content_type, content_id): - self.set_mock_return_value("mark_thread_as_read", {}) + self.set_mock_return_value('mark_thread_as_read', {}) def register_delete_thread_response(self, thread_id): - self.set_mock_return_value("delete_thread", {}) + self.set_mock_return_value('delete_thread', {}) def register_delete_comment_response(self, comment_id): - self.set_mock_return_value("delete_comment", {}) + self.set_mock_return_value('delete_comment', {}) def register_user_active_threads(self, user_id, response): - self.set_mock_return_value("get_user_active_threads", response) + self.set_mock_return_value('get_user_active_threads', response) def register_get_subscriptions(self, thread_id, response): - self.set_mock_return_value("get_thread_subscriptions", response) + self.set_mock_return_value('get_thread_subscriptions', response) def register_thread_flag_response(self, thread_id): """Register a mock response for PUT on the CS thread flag endpoints""" @@ -806,7 +760,7 @@ def request_patch(self, request_data): return self.client.patch( self.url, json.dumps(request_data), - content_type="application/merge-patch+json", + content_type="application/merge-patch+json" ) def expected_thread_data(self, overrides=None): @@ -864,10 +818,6 @@ def expected_thread_data(self, overrides=None): "close_reason": None, "close_reason_code": None, "learner_status": "new", - "is_deleted": None, - "deleted_at": None, - "deleted_by": None, - "deleted_by_label": None, } response_data.update(overrides or {}) return response_data @@ -940,9 +890,7 @@ def make_minimal_cs_comment(overrides=None): return ret -def make_paginated_api_response( - results=None, count=0, num_pages=0, next_link=None, previous_link=None -): +def make_paginated_api_response(results=None, count=0, num_pages=0, next_link=None, previous_link=None): """ Generates the response dictionary of paginated APIs with passed data """ @@ -953,7 +901,7 @@ def make_paginated_api_response( "count": count, "num_pages": num_pages, }, - "results": results or [], + "results": results or [] } @@ -971,9 +919,7 @@ def create_profile_image(self, user, storage): with make_image_file() as image_file: create_profile_images(image_file, get_profile_image_names(user.username)) self.check_images(user, storage) - set_has_profile_image( - user.username, True, self.TEST_PROFILE_IMAGE_UPLOADED_AT - ) + set_has_profile_image(user.username, True, self.TEST_PROFILE_IMAGE_UPLOADED_AT) def check_images(self, user, storage, exist=True): """ @@ -987,7 +933,7 @@ def check_images(self, user, storage, exist=True): assert storage.exists(name) with closing(Image.open(storage.path(name))) as img: assert img.size == (size, size) - assert img.format == "JPEG" + assert img.format == 'JPEG' else: assert not storage.exists(name) @@ -995,18 +941,18 @@ def get_expected_user_profile(self, username): """ Returns the expected user profile data for a given username """ - url = "http://example-storage.com/profile-images/{filename}_{{size}}.jpg?v={timestamp}".format( - filename=hashlib.md5(b"secret" + username.encode("utf-8")).hexdigest(), - timestamp=self.TEST_PROFILE_IMAGE_UPLOADED_AT.strftime("%s"), + url = 'http://example-storage.com/profile-images/{filename}_{{size}}.jpg?v={timestamp}'.format( + filename=hashlib.md5(b'secret' + username.encode('utf-8')).hexdigest(), + timestamp=self.TEST_PROFILE_IMAGE_UPLOADED_AT.strftime("%s") ) return { - "profile": { - "image": { - "has_image": True, - "image_url_full": url.format(size=500), - "image_url_large": url.format(size=120), - "image_url_medium": url.format(size=50), - "image_url_small": url.format(size=30), + 'profile': { + 'image': { + 'has_image': True, + 'image_url_full': url.format(size=500), + 'image_url_large': url.format(size=120), + 'image_url_medium': url.format(size=50), + 'image_url_small': url.format(size=30), } } } @@ -1016,14 +962,14 @@ def parsed_body(request): """Returns a parsed dictionary version of a request body""" # This could just be HTTPrettyRequest.parsed_body, but that method double-decodes '%2B' -> '+' -> ' '. # You can just remove this method when this issue is fixed: https://github.com/gabrielfalcao/HTTPretty/issues/240 - return parse_qs(request.body.decode("utf8")) + return parse_qs(request.body.decode('utf8')) def querystring(request): """Returns a parsed dictionary version of a query string""" # This could just be HTTPrettyRequest.querystring, but that method double-decodes '%2B' -> '+' -> ' '. # You can just remove this method when this issue is fixed: https://github.com/gabrielfalcao/HTTPretty/issues/240 - return parse_qs(request.path.split("?", 1)[-1]) + return parse_qs(request.path.split('?', 1)[-1]) class ThreadMock(object): @@ -1031,9 +977,7 @@ class ThreadMock(object): A mock thread object """ - def __init__( - self, thread_id, creator, title, parent_id=None, body="", commentable_id=None - ): + def __init__(self, thread_id, creator, title, parent_id=None, body='', commentable_id=None): self.id = thread_id self.user_id = str(creator.id) self.username = creator.username diff --git a/lms/djangoapps/discussion/rest_api/urls.py b/lms/djangoapps/discussion/rest_api/urls.py index 9753774f075c..f102dc41f249 100644 --- a/lms/djangoapps/discussion/rest_api/urls.py +++ b/lms/djangoapps/discussion/rest_api/urls.py @@ -9,7 +9,6 @@ from lms.djangoapps.discussion.rest_api.views import ( BulkDeleteUserPosts, - BulkRestoreUserPosts, CommentViewSet, CourseActivityStatsView, CourseDiscussionRolesAPIView, @@ -19,10 +18,8 @@ CourseTopicsViewV3, CourseView, CourseViewV2, - DeletedContentView, LearnerThreadView, ReplaceUsernamesView, - RestoreContent, RetireUserView, ThreadViewSet, UploadFileView, @@ -34,22 +31,26 @@ urlpatterns = [ re_path( - r"^v1/courses/{}/settings$".format(settings.COURSE_ID_PATTERN), + r"^v1/courses/{}/settings$".format( + settings.COURSE_ID_PATTERN + ), CourseDiscussionSettingsAPIView.as_view(), name="discussion_course_settings", ), re_path( - r"^v1/courses/{}/learner/$".format(settings.COURSE_ID_PATTERN), + r"^v1/courses/{}/learner/$".format( + settings.COURSE_ID_PATTERN + ), LearnerThreadView.as_view(), name="discussion_learner_threads", ), re_path( - rf"^v1/courses/{settings.COURSE_KEY_PATTERN}/activity_stats", + fr"^v1/courses/{settings.COURSE_KEY_PATTERN}/activity_stats", CourseActivityStatsView.as_view(), name="discussion_course_activity_stats", ), re_path( - rf"^v1/courses/{settings.COURSE_ID_PATTERN}/upload$", + fr"^v1/courses/{settings.COURSE_ID_PATTERN}/upload$", UploadFileView.as_view(), name="upload_file", ), @@ -61,55 +62,36 @@ name="discussion_course_roles", ), re_path( - rf"^v1/courses/{settings.COURSE_ID_PATTERN}", + fr"^v1/courses/{settings.COURSE_ID_PATTERN}", CourseView.as_view(), - name="discussion_course", + name="discussion_course" ), re_path( - rf"^v2/courses/{settings.COURSE_ID_PATTERN}", + fr"^v2/courses/{settings.COURSE_ID_PATTERN}", CourseViewV2.as_view(), - name="discussion_course_v2", + name="discussion_course_v2" ), + re_path(r'^v1/accounts/retire_forum/?$', RetireUserView.as_view(), name="retire_discussion_user"), + path('v1/accounts/replace_username', ReplaceUsernamesView.as_view(), name="replace_discussion_username"), re_path( - r"^v1/accounts/retire_forum/?$", - RetireUserView.as_view(), - name="retire_discussion_user", - ), - path( - "v1/accounts/replace_username", - ReplaceUsernamesView.as_view(), - name="replace_discussion_username", - ), - re_path( - rf"^v1/course_topics/{settings.COURSE_ID_PATTERN}", + fr"^v1/course_topics/{settings.COURSE_ID_PATTERN}", CourseTopicsView.as_view(), - name="course_topics", + name="course_topics" ), re_path( - rf"^v2/course_topics/{settings.COURSE_ID_PATTERN}", + fr"^v2/course_topics/{settings.COURSE_ID_PATTERN}", CourseTopicsViewV2.as_view(), - name="course_topics_v2", + name="course_topics_v2" ), re_path( - rf"^v3/course_topics/{settings.COURSE_ID_PATTERN}", + fr"^v3/course_topics/{settings.COURSE_ID_PATTERN}", CourseTopicsViewV3.as_view(), - name="course_topics_v3", + name="course_topics_v3" ), re_path( - rf"^v1/bulk_delete_user_posts/{settings.COURSE_ID_PATTERN}", + fr"^v1/bulk_delete_user_posts/{settings.COURSE_ID_PATTERN}", BulkDeleteUserPosts.as_view(), - name="bulk_delete_user_posts", - ), - re_path( - rf"^v1/bulk_restore_user_posts/{settings.COURSE_ID_PATTERN}", - BulkRestoreUserPosts.as_view(), - name="bulk_restore_user_posts", - ), - path("v1/restore_content", RestoreContent.as_view(), name="restore_content"), - re_path( - rf"^v1/deleted_content/{settings.COURSE_ID_PATTERN}", - DeletedContentView.as_view(), - name="deleted_content", + name="bulk_delete_user_posts" ), - path("v1/", include(ROUTER.urls)), + path('v1/', include(ROUTER.urls)), ] diff --git a/lms/djangoapps/discussion/rest_api/views.py b/lms/djangoapps/discussion/rest_api/views.py index 3556f78562fe..ba9818124e08 100644 --- a/lms/djangoapps/discussion/rest_api/views.py +++ b/lms/djangoapps/discussion/rest_api/views.py @@ -1,19 +1,17 @@ """ Discussion API views """ - import logging import uuid import edx_api_doc_tools as apidocs + from django.contrib.auth import get_user_model from django.core.exceptions import BadRequest, ValidationError from django.shortcuts import get_object_or_404 from drf_yasg import openapi from edx_rest_framework_extensions.auth.jwt.authentication import JwtAuthentication -from edx_rest_framework_extensions.auth.session.authentication import ( - SessionAuthenticationAllowInactiveUser, -) +from edx_rest_framework_extensions.auth.session.authentication import SessionAuthenticationAllowInactiveUser from opaque_keys.edx.keys import CourseKey from rest_framework import permissions, status from rest_framework.authentication import SessionAuthentication @@ -23,49 +21,31 @@ from rest_framework.views import APIView from rest_framework.viewsets import ViewSet +from xmodule.modulestore.django import modulestore + from common.djangoapps.student.models import CourseEnrollment from common.djangoapps.util.file import store_uploaded_file from lms.djangoapps.course_api.blocks.api import get_blocks from lms.djangoapps.course_goals.models import UserActivity -from lms.djangoapps.discussion.django_comment_client import settings as cc_settings -from lms.djangoapps.discussion.django_comment_client.utils import ( - get_group_id_for_comments_service, -) from lms.djangoapps.discussion.rate_limit import is_content_creation_rate_limited from lms.djangoapps.discussion.rest_api.permissions import IsAllowedToBulkDelete -from lms.djangoapps.discussion.rest_api.tasks import ( - delete_course_post_for_user, - restore_course_post_for_user, -) +from lms.djangoapps.discussion.rest_api.tasks import delete_course_post_for_user from lms.djangoapps.discussion.toggles import ONLY_VERIFIED_USERS_CAN_POST +from lms.djangoapps.discussion.django_comment_client import settings as cc_settings +from lms.djangoapps.discussion.django_comment_client.utils import get_group_id_for_comments_service from lms.djangoapps.instructor.access import update_forum_role -from openedx.core.djangoapps.discussions.config.waffle import ( - ENABLE_NEW_STRUCTURE_DISCUSSIONS, -) -from openedx.core.djangoapps.discussions.models import ( - DiscussionsConfiguration, - Provider, -) +from openedx.core.djangoapps.discussions.config.waffle import ENABLE_NEW_STRUCTURE_DISCUSSIONS +from openedx.core.djangoapps.discussions.models import DiscussionsConfiguration, Provider from openedx.core.djangoapps.discussions.serializers import DiscussionSettingsSerializer from openedx.core.djangoapps.django_comment_common import comment_client +from openedx.core.djangoapps.django_comment_common.models import CourseDiscussionSettings, Role from openedx.core.djangoapps.django_comment_common.comment_client.comment import Comment from openedx.core.djangoapps.django_comment_common.comment_client.thread import Thread -from openedx.core.djangoapps.django_comment_common.models import ( - CourseDiscussionSettings, - Role, -) -from openedx.core.djangoapps.user_api.accounts.permissions import ( - CanReplaceUsername, - CanRetireUser, -) +from openedx.core.djangoapps.user_api.accounts.permissions import CanReplaceUsername, CanRetireUser from openedx.core.djangoapps.user_api.models import UserRetirementStatus -from openedx.core.lib.api.authentication import ( - BearerAuthentication, - BearerAuthenticationAllowInactiveUser, -) +from openedx.core.lib.api.authentication import BearerAuthentication, BearerAuthenticationAllowInactiveUser from openedx.core.lib.api.parsers import MergePatchParser from openedx.core.lib.api.view_utils import DeveloperErrorViewMixin, view_auth_classes -from xmodule.modulestore.django import modulestore from ..rest_api.api import ( create_comment, @@ -77,10 +57,10 @@ get_course_discussion_user_stats, get_course_topics, get_course_topics_v2, - get_learner_active_thread_list, get_response_comments, get_thread, get_thread_list, + get_learner_active_thread_list, get_user_comments, get_v2_course_topics_as_v1, update_comment, @@ -108,10 +88,10 @@ from .utils import ( create_blocks_params, create_topics_v3_structure, - get_course_id_from_thread_id, is_captcha_enabled, - is_only_student, verify_recaptcha_token, + get_course_id_from_thread_id, + is_only_student, ) log = logging.getLogger(__name__) @@ -127,16 +107,14 @@ class CourseView(DeveloperErrorViewMixin, APIView): @apidocs.schema( parameters=[ - apidocs.string_parameter( - "course_id", apidocs.ParameterLocation.PATH, description="Course ID" - ) + apidocs.string_parameter("course_id", apidocs.ParameterLocation.PATH, description="Course ID") ], responses={ 200: CourseMetadataSerailizer(read_only=True, required=False), 401: "The requester is not authenticated.", 403: "The requester cannot access the specified course.", 404: "The requested course does not exist.", - }, + } ) def get(self, request, course_id): """ @@ -148,9 +126,7 @@ def get(self, request, course_id): """ course_key = CourseKey.from_string(course_id) # TODO: which class is right? # Record user activity for tracking progress towards a user's course goals (for mobile app) - UserActivity.record_user_activity( - request.user, course_key, request=request, only_if_mobile_app=True - ) + UserActivity.record_user_activity(request.user, course_key, request=request, only_if_mobile_app=True) return Response(get_course(request, course_key)) @@ -162,16 +138,14 @@ class CourseViewV2(DeveloperErrorViewMixin, APIView): @apidocs.schema( parameters=[ - apidocs.string_parameter( - "course_id", apidocs.ParameterLocation.PATH, description="Course ID" - ) + apidocs.string_parameter("course_id", apidocs.ParameterLocation.PATH, description="Course ID") ], responses={ 200: CourseMetadataSerailizer(read_only=True, required=False), 401: "The requester is not authenticated.", 403: "The requester cannot access the specified course.", 404: "The requested course does not exist.", - }, + } ) def get(self, request, course_id): """ @@ -182,9 +156,7 @@ def get(self, request, course_id): """ course_key = CourseKey.from_string(course_id) # Record user activity for tracking progress towards a user's course goals (for mobile app) - UserActivity.record_user_activity( - request.user, course_key, request=request, only_if_mobile_app=True - ) + UserActivity.record_user_activity(request.user, course_key, request=request, only_if_mobile_app=True) return Response(get_course(request, course_key, False)) @@ -249,14 +221,14 @@ def get(self, request, course_key_string): form_query_string = CourseActivityStatsForm(request.query_params) if not form_query_string.is_valid(): raise ValidationError(form_query_string.errors) - order_by = form_query_string.cleaned_data.get("order_by", None) + order_by = form_query_string.cleaned_data.get('order_by', None) order_by = UserOrdering(order_by) if order_by else None - username_search_string = form_query_string.cleaned_data.get("username", None) + username_search_string = form_query_string.cleaned_data.get('username', None) data = get_course_discussion_user_stats( request, course_key_string, - form_query_string.cleaned_data["page"], - form_query_string.cleaned_data["page_size"], + form_query_string.cleaned_data['page'], + form_query_string.cleaned_data['page_size'], order_by, username_search_string, ) @@ -296,17 +268,19 @@ def get(self, request, course_id): Implements the GET method as described in the class docstring. """ course_key = CourseKey.from_string(course_id) - topic_ids = self.request.GET.get("topic_id") - topic_ids = set(topic_ids.strip(",").split(",")) if topic_ids else None + topic_ids = self.request.GET.get('topic_id') + topic_ids = set(topic_ids.strip(',').split(',')) if topic_ids else None with modulestore().bulk_operations(course_key): configuration = DiscussionsConfiguration.get(context_key=course_key) provider = configuration.provider_type # This will be removed when mobile app will support new topic structure - new_structure_enabled = ENABLE_NEW_STRUCTURE_DISCUSSIONS.is_enabled( - course_key - ) + new_structure_enabled = ENABLE_NEW_STRUCTURE_DISCUSSIONS.is_enabled(course_key) if provider == Provider.OPEN_EDX and new_structure_enabled: - response = get_v2_course_topics_as_v1(request, course_key, topic_ids) + response = get_v2_course_topics_as_v1( + request, + course_key, + topic_ids + ) else: response = get_course_topics( request, @@ -314,9 +288,7 @@ def get(self, request, course_id): topic_ids, ) # Record user activity for tracking progress towards a user's course goals (for mobile app) - UserActivity.record_user_activity( - request.user, course_key, request=request, only_if_mobile_app=True - ) + UserActivity.record_user_activity(request.user, course_key, request=request, only_if_mobile_app=True) return Response(response) @@ -332,17 +304,17 @@ class CourseTopicsViewV2(DeveloperErrorViewMixin, APIView): @apidocs.schema( parameters=[ apidocs.string_parameter( - "course_id", + 'course_id', apidocs.ParameterLocation.PATH, description="Course ID", ), apidocs.string_parameter( - "topic_id", + 'topic_id', apidocs.ParameterLocation.QUERY, description="Comma-separated list of topic ids to filter", ), openapi.Parameter( - "order_by", + 'order_by', apidocs.ParameterLocation.QUERY, required=False, type=openapi.TYPE_STRING, @@ -355,7 +327,7 @@ class CourseTopicsViewV2(DeveloperErrorViewMixin, APIView): 401: "The requester is not authenticated.", 403: "The requester cannot access the specified course.", 404: "The requested course does not exist.", - }, + } ) def get(self, request, course_id): """ @@ -376,7 +348,7 @@ def get(self, request, course_id): course_key, request.user, form_query_params.cleaned_data["topic_id"], - form_query_params.cleaned_data["order_by"], + form_query_params.cleaned_data["order_by"] ) return Response(response) @@ -444,17 +416,17 @@ def get(self, request, course_id): blocks_params = create_blocks_params(course_usage_key, request.user) blocks = get_blocks( request, - blocks_params["usage_key"], - blocks_params["user"], - blocks_params["depth"], - blocks_params["nav_depth"], - blocks_params["requested_fields"], - blocks_params["block_counts"], - blocks_params["student_view_data"], - blocks_params["return_type"], - blocks_params["block_types_filter"], + blocks_params['usage_key'], + blocks_params['user'], + blocks_params['depth'], + blocks_params['nav_depth'], + blocks_params['requested_fields'], + blocks_params['block_counts'], + blocks_params['student_view_data'], + blocks_params['return_type'], + blocks_params['block_types_filter'], hide_access_denials=False, - )["blocks"] + )['blocks'] topics = create_topics_v3_structure(blocks, topics) return Response(topics) @@ -655,12 +627,8 @@ class ThreadViewSet(DeveloperErrorViewMixin, ViewSet): No content is returned for a DELETE request """ - lookup_field = "thread_id" - parser_classes = ( - JSONParser, - MergePatchParser, - ) + parser_classes = (JSONParser, MergePatchParser,) def list(self, request): """ @@ -673,10 +641,7 @@ class docstring. # Record user activity for tracking progress towards a user's course goals (for mobile app) UserActivity.record_user_activity( - request.user, - form.cleaned_data["course_id"], - request=request, - only_if_mobile_app=True, + request.user, form.cleaned_data["course_id"], request=request, only_if_mobile_app=True ) return get_thread_list( @@ -695,15 +660,14 @@ class docstring. form.cleaned_data["order_direction"], form.cleaned_data["requested_fields"], form.cleaned_data["count_flagged"], - form.cleaned_data["show_deleted"], ) def retrieve(self, request, thread_id=None): """ Implements the GET method for thread ID """ - requested_fields = request.GET.get("requested_fields") - course_id = request.GET.get("course_id") + requested_fields = request.GET.get('requested_fields') + course_id = request.GET.get('course_id') return Response(get_thread(request, thread_id, requested_fields, course_id)) def create(self, request): @@ -717,28 +681,21 @@ class docstring. course_key = CourseKey.from_string(course_key_str) if is_content_creation_rate_limited(request, course_key=course_key): - return Response( - "Too many requests", status=status.HTTP_429_TOO_MANY_REQUESTS - ) + return Response("Too many requests", status=status.HTTP_429_TOO_MANY_REQUESTS) if is_captcha_enabled(course_key) and is_only_student(course_key, request.user): - captcha_token = request.data.get("captcha_token") + captcha_token = request.data.get('captcha_token') if not captcha_token: - raise ValidationError({"captcha_token": "This field is required."}) + raise ValidationError({'captcha_token': 'This field is required.'}) if not verify_recaptcha_token(captcha_token): - return Response({"error": "CAPTCHA verification failed."}, status=400) - - if ( - ONLY_VERIFIED_USERS_CAN_POST.is_enabled(course_key) - and not request.user.is_active - ): - raise ValidationError( - {"detail": "Only verified users can post in discussions."} - ) + return Response({'error': 'CAPTCHA verification failed.'}, status=400) + + if ONLY_VERIFIED_USERS_CAN_POST.is_enabled(course_key) and not request.user.is_active: + raise ValidationError({"detail": "Only verified users can post in discussions."}) data = request.data.copy() - data.pop("captcha_token", None) + data.pop('captcha_token', None) return Response(create_thread(request, data)) def partial_update(self, request, thread_id): @@ -805,27 +762,24 @@ def get(self, request, course_id=None): Implements the GET method as described in the class docstring. """ course_key = CourseKey.from_string(course_id) - page_num = request.GET.get("page", 1) - threads_per_page = request.GET.get("page_size", 10) - count_flagged = request.GET.get("count_flagged", False) - thread_type = request.GET.get("thread_type") - order_by = request.GET.get("order_by") + page_num = request.GET.get('page', 1) + threads_per_page = request.GET.get('page_size', 10) + count_flagged = request.GET.get('count_flagged', False) + thread_type = request.GET.get('thread_type') + order_by = request.GET.get('order_by') order_by_mapping = { "last_activity_at": "activity", "comment_count": "comments", - "vote_count": "votes", + "vote_count": "votes" } - order_by = order_by_mapping.get(order_by, "activity") - post_status = request.GET.get("status", None) - show_deleted = request.GET.get("show_deleted", "false").lower() == "true" + order_by = order_by_mapping.get(order_by, 'activity') + post_status = request.GET.get('status', None) discussion_id = None - username = request.GET.get("username", None) + username = request.GET.get('username', None) user = get_object_or_404(User, username=username) group_id = None try: - group_id = get_group_id_for_comments_service( - request, course_key, discussion_id - ) + group_id = get_group_id_for_comments_service(request, course_key, discussion_id) except ValueError: pass @@ -838,17 +792,14 @@ def get(self, request, course_id=None): "count_flagged": count_flagged, "thread_type": thread_type, "sort_key": order_by, - "show_deleted": show_deleted, } if post_status: - if post_status not in ["flagged", "unanswered", "unread", "unresponded"]: - raise ValidationError( - { - "status": [ - f"Invalid value. '{post_status}' must be 'flagged', 'unanswered', 'unread' or 'unresponded" - ] - } - ) + if post_status not in ['flagged', 'unanswered', 'unread', 'unresponded']: + raise ValidationError({ + "status": [ + f"Invalid value. '{post_status}' must be 'flagged', 'unanswered', 'unread' or 'unresponded" + ] + }) query_params[post_status] = True return get_learner_active_thread_list(request, course_key, query_params) @@ -1017,12 +968,8 @@ class CommentViewSet(DeveloperErrorViewMixin, ViewSet): No content is returned for a DELETE request """ - lookup_field = "comment_id" - parser_classes = ( - JSONParser, - MergePatchParser, - ) + parser_classes = (JSONParser, MergePatchParser,) def list(self, request): """ @@ -1063,8 +1010,7 @@ def list_by_thread(self, request): form.cleaned_data["page_size"], form.cleaned_data["flagged"], form.cleaned_data["requested_fields"], - form.cleaned_data["merge_question_type_responses"], - form.cleaned_data["show_deleted"], + form.cleaned_data["merge_question_type_responses"] ) def list_by_user(self, request): @@ -1111,28 +1057,21 @@ class docstring. course_key = CourseKey.from_string(course_key_str) if is_content_creation_rate_limited(request, course_key=course_key): - return Response( - "Too many requests", status=status.HTTP_429_TOO_MANY_REQUESTS - ) + return Response("Too many requests", status=status.HTTP_429_TOO_MANY_REQUESTS) if is_captcha_enabled(course_key) and is_only_student(course_key, request.user): - captcha_token = request.data.get("captcha_token") + captcha_token = request.data.get('captcha_token') if not captcha_token: - raise ValidationError({"captcha_token": "This field is required."}) + raise ValidationError({'captcha_token': 'This field is required.'}) if not verify_recaptcha_token(captcha_token): - return Response({"error": "CAPTCHA verification failed."}, status=400) - - if ( - ONLY_VERIFIED_USERS_CAN_POST.is_enabled(course_key) - and not request.user.is_active - ): - raise ValidationError( - {"detail": "Only verified users can post in discussions."} - ) + return Response({'error': 'CAPTCHA verification failed.'}, status=400) + + if ONLY_VERIFIED_USERS_CAN_POST.is_enabled(course_key) and not request.user.is_active: + raise ValidationError({"detail": "Only verified users can post in discussions."}) data = request.data.copy() - data.pop("captcha_token", None) + data.pop('captcha_token', None) return Response(create_comment(request, data)) def destroy(self, request, comment_id): @@ -1208,11 +1147,8 @@ def post(self, request, course_id): unique_file_name = f"{course_id}/{thread_key}/{uuid.uuid4()}" try: file_storage, stored_file_name = store_uploaded_file( - request, - "uploaded_file", - cc_settings.ALLOWED_UPLOAD_FILE_TYPES, - unique_file_name, - max_file_size=cc_settings.MAX_UPLOAD_FILE_SIZE, + request, "uploaded_file", cc_settings.ALLOWED_UPLOAD_FILE_TYPES, + unique_file_name, max_file_size=cc_settings.MAX_UPLOAD_FILE_SIZE, ) except ValueError as err: raise BadRequest("no `uploaded_file` was provided") from err @@ -1253,12 +1189,10 @@ def post(self, request): """ Implements the retirement endpoint. """ - username = request.data["username"] + username = request.data['username'] try: - retirement = UserRetirementStatus.get_retirement_for_retirement_action( - username - ) + retirement = UserRetirementStatus.get_retirement_for_retirement_action(username) cc_user = comment_client.User.from_django_user(retirement.user) # Send the retired username to the forums service, as the service cannot generate @@ -1313,9 +1247,7 @@ def post(self, request): for username_pair in username_mappings: current_username = list(username_pair.keys())[0] new_username = list(username_pair.values())[0] - successfully_replaced = self._replace_username( - current_username, new_username - ) + successfully_replaced = self._replace_username(current_username, new_username) if successfully_replaced: successful_replacements.append({current_username: new_username}) else: @@ -1325,8 +1257,8 @@ def post(self, request): status=status.HTTP_200_OK, data={ "successful_replacements": successful_replacements, - "failed_replacements": failed_replacements, - }, + "failed_replacements": failed_replacements + } ) def _replace_username(self, current_username, new_username): @@ -1372,7 +1304,7 @@ def _replace_username(self, current_username, new_username): return True def _has_valid_schema(self, post_data): - """Verifies the data is a list of objects with a single key:value pair""" + """ Verifies the data is a list of objects with a single key:value pair """ if not isinstance(post_data, list): return False for obj in post_data: @@ -1432,16 +1364,12 @@ class CourseDiscussionSettingsAPIView(DeveloperErrorViewMixin, APIView): * available_division_schemes: A list of available division schemes for the course. """ - authentication_classes = ( JwtAuthentication, BearerAuthenticationAllowInactiveUser, SessionAuthenticationAllowInactiveUser, ) - parser_classes = ( - JSONParser, - MergePatchParser, - ) + parser_classes = (JSONParser, MergePatchParser,) permission_classes = (permissions.IsAuthenticated, IsStaffOrAdmin) def _get_request_kwargs(self, course_id): @@ -1457,14 +1385,14 @@ def get(self, request, course_id): if not form.is_valid(): raise ValidationError(form.errors) - course_key = form.cleaned_data["course_key"] - course = form.cleaned_data["course"] + course_key = form.cleaned_data['course_key'] + course = form.cleaned_data['course'] discussion_settings = CourseDiscussionSettings.get(course_key) serializer = DiscussionSettingsSerializer( discussion_settings, context={ - "course": course, - "settings": discussion_settings, + 'course': course, + 'settings': discussion_settings, }, partial=True, ) @@ -1483,15 +1411,15 @@ def patch(self, request, course_id): if not form.is_valid(): raise ValidationError(form.errors) - course = form.cleaned_data["course"] - course_key = form.cleaned_data["course_key"] + course = form.cleaned_data['course'] + course_key = form.cleaned_data['course_key'] discussion_settings = CourseDiscussionSettings.get(course_key) serializer = DiscussionSettingsSerializer( discussion_settings, context={ - "course": course, - "settings": discussion_settings, + 'course': course, + 'settings': discussion_settings, }, data=request.data, partial=True, @@ -1560,7 +1488,6 @@ class CourseDiscussionRolesAPIView(DeveloperErrorViewMixin, APIView): * division_scheme: The division scheme used by the course. """ - authentication_classes = ( JwtAuthentication, BearerAuthenticationAllowInactiveUser, @@ -1581,13 +1508,11 @@ def get(self, request, course_id, rolename): if not form.is_valid(): raise ValidationError(form.errors) - course_id = form.cleaned_data["course_key"] - role = form.cleaned_data["role"] + course_id = form.cleaned_data['course_key'] + role = form.cleaned_data['role'] - data = {"course_id": course_id, "users": role.users.all()} - context = { - "course_discussion_settings": CourseDiscussionSettings.get(course_id) - } + data = {'course_id': course_id, 'users': role.users.all()} + context = {'course_discussion_settings': CourseDiscussionSettings.get(course_id)} serializer = DiscussionRolesListSerializer(data, context=context) return Response(serializer.data) @@ -1601,25 +1526,23 @@ def post(self, request, course_id, rolename): if not form.is_valid(): raise ValidationError(form.errors) - course_id = form.cleaned_data["course_key"] - rolename = form.cleaned_data["rolename"] + course_id = form.cleaned_data['course_key'] + rolename = form.cleaned_data['rolename'] serializer = DiscussionRolesSerializer(data=request.data) if not serializer.is_valid(): raise ValidationError(serializer.errors) - action = serializer.validated_data["action"] - user = serializer.validated_data["user"] + action = serializer.validated_data['action'] + user = serializer.validated_data['user'] try: update_forum_role(course_id, user, rolename, action) except Role.DoesNotExist as err: raise ValidationError(f"Role '{rolename}' does not exist") from err - role = form.cleaned_data["role"] - data = {"course_id": course_id, "users": role.users.all()} - context = { - "course_discussion_settings": CourseDiscussionSettings.get(course_id) - } + role = form.cleaned_data['role'] + data = {'course_id': course_id, 'users': role.users.all()} + context = {'course_discussion_settings': CourseDiscussionSettings.get(course_id)} serializer = DiscussionRolesListSerializer(data, context=context) return Response(serializer.data) @@ -1643,9 +1566,7 @@ class BulkDeleteUserPosts(DeveloperErrorViewMixin, APIView): """ authentication_classes = ( - JwtAuthentication, - BearerAuthentication, - SessionAuthentication, + JwtAuthentication, BearerAuthentication, SessionAuthentication, ) permission_classes = (permissions.IsAuthenticated, IsAllowedToBulkDelete) @@ -1666,26 +1587,23 @@ def post(self, request, course_id): course_ids = [course_id] if course_or_org == "org": org_id = CourseKey.from_string(course_id).org - enrollments = CourseEnrollment.objects.filter( - user=request.user - ).values_list("course_id", flat=True) - course_ids.extend([str(c_id) for c_id in enrollments if c_id.org == org_id]) + enrollments = CourseEnrollment.objects.filter(user=request.user).values_list('course_id', flat=True) + course_ids.extend([ + str(c_id) + for c_id in enrollments + if c_id.org == org_id + ]) course_ids = list(set(course_ids)) log.info(f"<> {username} enrolled in {enrollments}") - log.info( - f"<> Posts for {username} in {course_ids} - for {course_or_org} {course_id}" - ) + log.info(f"<> Posts for {username} in {course_ids} - for {course_or_org} {course_id}") comment_count = Comment.get_user_comment_count(user.id, course_ids) thread_count = Thread.get_user_threads_count(user.id, course_ids) - log.info( - f"<> {username} in {course_ids} - Count thread {thread_count}, comment {comment_count}" - ) + log.info(f"<> {username} in {course_ids} - Count thread {thread_count}, comment {comment_count}") if execute_task: event_data = { "triggered_by": request.user.username, - "triggered_by_user_id": str(request.user.id), "username": username, "course_or_org": course_or_org, "course_key": course_id, @@ -1695,256 +1613,5 @@ def post(self, request, course_id): ) return Response( {"comment_count": comment_count, "thread_count": thread_count}, - status=status.HTTP_202_ACCEPTED, - ) - - -class RestoreContent(DeveloperErrorViewMixin, APIView): - """ - **Use Cases** - A privileged user that can restore individual soft-deleted threads, comments, or responses. - - **Example Requests**: - POST /api/discussion/v1/restore_content - Request Body: - { - "content_type": "thread", // "thread", "comment", or "response" - "content_id": "thread_id_or_comment_id", - "course_id": "course-v1:edX+DemoX+Demo_Course" - } - - **Example Response**: - {"success": true, "message": "Content restored successfully"} - """ - - authentication_classes = ( - JwtAuthentication, - BearerAuthentication, - SessionAuthentication, - ) - permission_classes = (permissions.IsAuthenticated, IsAllowedToBulkDelete) - - def post(self, request): - """ - Implements the restore individual content endpoint. - """ - content_type = request.data.get("content_type") - content_id = request.data.get("content_id") - course_id = request.data.get("course_id") - - if not all([content_type, content_id, course_id]): - raise BadRequest("content_type, content_id, and course_id are required.") - - if content_type not in ["thread", "comment", "response"]: - raise BadRequest("content_type must be 'thread', 'comment', or 'response'.") - - restored_by_user_id = str(request.user.id) - - try: - if content_type == "thread": - success = Thread.restore_thread( - content_id, course_id=course_id, restored_by=restored_by_user_id - ) - else: # comment or response (both are comments in the backend) - success = Comment.restore_comment( - content_id, course_id=course_id, restored_by=restored_by_user_id - ) - - if success: - return Response( - { - "success": True, - "message": f"{content_type.capitalize()} restored successfully", - }, - status=status.HTTP_200_OK, - ) - else: - return Response( - { - "success": False, - "message": f"{content_type.capitalize()} not found or already restored", - }, - status=status.HTTP_404_NOT_FOUND, - ) - except Exception as e: # pylint: disable=broad-exception-caught - log.error("Error restoring %s %s: %s", content_type, content_id, str(e)) - return Response( - { - "success": False, - "message": f"Error restoring {content_type}: {str(e)}", - }, - status=status.HTTP_500_INTERNAL_SERVER_ERROR, - ) - - -class BulkRestoreUserPosts(DeveloperErrorViewMixin, APIView): - """ - **Use Cases** - A privileged user that can restore all soft-deleted posts and comments made by a user. - It returns expected number of comments and threads that will be restored - - **Example Requests**: - POST /api/discussion/v1/bulk_restore_user_posts/{course_id} - Query Parameters: - username: The username of the user whose posts are to be restored - course_id: Course id for which posts are to be restored - execute: If True, runs restoration task - course_or_org: If 'course', restores posts in the course, if 'org', restores posts in all courses of the org - - **Example Response**: - {"comment_count": 5, "thread_count": 3} - """ - - authentication_classes = ( - JwtAuthentication, - BearerAuthentication, - SessionAuthentication, - ) - permission_classes = (permissions.IsAuthenticated, IsAllowedToBulkDelete) - - def post(self, request, course_id): - """ - Implements the restore user posts endpoint. - """ - username = request.GET.get("username", None) - execute_task = request.GET.get("execute", "false").lower() == "true" - if (not username) or (not course_id): - raise BadRequest("username and course_id are required.") - course_or_org = request.GET.get("course_or_org", "course") - if course_or_org not in ["course", "org"]: - raise BadRequest("course_or_org must be either 'course' or 'org'.") - - user = get_object_or_404(User, username=username) - course_ids = [course_id] - if course_or_org == "org": - org_id = CourseKey.from_string(course_id).org - enrollments = CourseEnrollment.objects.filter( - user=request.user - ).values_list("course_id", flat=True) - course_ids.extend([str(c_id) for c_id in enrollments if c_id.org == org_id]) - course_ids = list(set(course_ids)) - log.info("<> %s enrolled in %s", username, enrollments) - log.info( - "<> Posts for %s in %s - for %s %s", - username, - course_ids, - course_or_org, - course_id, - ) - - comment_count = Comment.get_user_deleted_comment_count(user.id, course_ids) - thread_count = Thread.get_user_deleted_threads_count(user.id, course_ids) - log.info( - "<> %s in %s - Count thread %s, comment %s", - username, - course_ids, - thread_count, - comment_count, - ) - - if execute_task: - event_data = { - "triggered_by": request.user.username, - "triggered_by_user_id": str(request.user.id), - "username": username, - "course_or_org": course_or_org, - "course_key": course_id, - } - restore_course_post_for_user.apply_async( - args=(user.id, username, course_ids, event_data), - ) - return Response( - {"comment_count": comment_count, "thread_count": thread_count}, - status=status.HTTP_202_ACCEPTED, + status=status.HTTP_202_ACCEPTED ) - - -class DeletedContentView(DeveloperErrorViewMixin, APIView): - """ - **Use Cases** - Retrieve all deleted content (threads, comments, responses) for a course. - This endpoint allows privileged users to fetch deleted discussion content. - - **Example Requests**: - GET /api/discussion/v1/deleted_content/course-v1:edX+DemoX+Demo_Course - GET /api/discussion/v1/deleted_content/course-v1:edX+DemoX+Demo_Course?content_type=thread - GET /api/discussion/v1/deleted_content/course-v1:edX+DemoX+Demo_Course?page=1&per_page=20 - - **Example Response**: - { - "results": [ - { - "id": "thread_id", - "type": "thread", - "title": "Deleted Thread Title", - "body": "Thread content...", - "course_id": "course-v1:edX+DemoX+Demo_Course", - "author_id": "user_123", - "deleted_at": "2023-11-19T10:30:00Z", - "deleted_by": "moderator_456" - } - ], - "pagination": { - "page": 1, - "per_page": 20, - "total_count": 50, - "num_pages": 3 - } - } - """ - - authentication_classes = ( - JwtAuthentication, - BearerAuthentication, - SessionAuthentication, - ) - permission_classes = (permissions.IsAuthenticated, IsAllowedToBulkDelete) - - def get(self, request, course_id): - """ - Retrieve all deleted content for a course. - """ - try: - course_key = CourseKey.from_string(course_id) - except Exception as e: - raise BadRequest("Invalid course_id") from e - - # Get query parameters - content_type = request.GET.get( - "content_type", None - ) # 'thread', 'comment', or None for all - page = int(request.GET.get("page", 1)) - per_page = int(request.GET.get("per_page", 20)) - author_id = request.GET.get("author_id", None) - - # Validate parameters - if content_type and content_type not in ["thread", "comment"]: - raise BadRequest("content_type must be 'thread' or 'comment'") - - per_page = min(per_page, 100) # Limit to prevent excessive load - - try: - # Import here to avoid circular imports - from lms.djangoapps.discussion.rest_api.api import ( - get_deleted_content_for_course, - ) - - results = get_deleted_content_for_course( - request=request, - course_id=str(course_key), - content_type=content_type, - page=page, - per_page=per_page, - author_id=author_id, - ) - - return Response(results, status=status.HTTP_200_OK) - - except Exception as e: # pylint: disable=broad-exception-caught - logging.exception( - "Error retrieving deleted content for course %s: %s", course_id, e - ) - return Response( - {"error": "Failed to retrieve deleted content"}, - status=status.HTTP_500_INTERNAL_SERVER_ERROR, - ) diff --git a/openedx/core/djangoapps/django_comment_common/comment_client/comment.py b/openedx/core/djangoapps/django_comment_common/comment_client/comment.py index dae2088594ae..8905679a45db 100644 --- a/openedx/core/djangoapps/django_comment_common/comment_client/comment.py +++ b/openedx/core/djangoapps/django_comment_common/comment_client/comment.py @@ -4,17 +4,13 @@ from bs4 import BeautifulSoup -from forum import api as forum_api -from forum.backends.mongodb.comments import ( - Comment as ForumComment, -) # pylint: disable=import-error -from openedx.core.djangoapps.django_comment_common.comment_client import ( - models, - settings, -) +from openedx.core.djangoapps.django_comment_common.comment_client import models, settings from .thread import Thread from .utils import CommentClientRequestError, get_course_key +from forum import api as forum_api +from forum.backends.mongodb.comments import Comment as ForumComment + log = logging.getLogger(__name__) @@ -22,56 +18,26 @@ class Comment(models.Model): accessible_fields = [ - "id", - "body", - "anonymous", - "anonymous_to_peers", - "course_id", - "endorsed", - "parent_id", - "thread_id", - "username", - "votes", - "user_id", - "closed", - "created_at", - "updated_at", - "depth", - "at_position_list", - "type", - "commentable_id", - "abuse_flaggers", - "endorsement", - "child_count", - "edit_history", - "is_spam", - "ai_moderation_reason", - "abuse_flagged", - "is_deleted", - "deleted_at", - "deleted_by", + 'id', 'body', 'anonymous', 'anonymous_to_peers', 'course_id', + 'endorsed', 'parent_id', 'thread_id', 'username', 'votes', 'user_id', + 'closed', 'created_at', 'updated_at', 'depth', 'at_position_list', + 'type', 'commentable_id', 'abuse_flaggers', 'endorsement', + 'child_count', 'edit_history', + 'is_spam', 'ai_moderation_reason', 'abuse_flagged', ] updatable_fields = [ - "body", - "anonymous", - "anonymous_to_peers", - "course_id", - "closed", - "user_id", - "endorsed", - "endorsement_user_id", - "edit_reason_code", - "closing_user_id", - "editing_user_id", + 'body', 'anonymous', 'anonymous_to_peers', 'course_id', 'closed', + 'user_id', 'endorsed', 'endorsement_user_id', 'edit_reason_code', + 'closing_user_id', 'editing_user_id', ] initializable_fields = updatable_fields - metrics_tag_fields = ["course_id", "endorsed", "closed"] + metrics_tag_fields = ['course_id', 'endorsed', 'closed'] base_url = f"{settings.PREFIX}/comments" - type = "comment" + type = 'comment' def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) @@ -80,7 +46,7 @@ def __init__(self, *args, **kwargs): @property def thread(self): if not self._cached_thread: - self._cached_thread = Thread(id=self.thread_id, type="thread") + self._cached_thread = Thread(id=self.thread_id, type='thread') return self._cached_thread @property @@ -90,22 +56,22 @@ def context(self): @classmethod def url_for_comments(cls, params=None): - if params and params.get("parent_id"): - return _url_for_comment(params["parent_id"]) + if params and params.get('parent_id'): + return _url_for_comment(params['parent_id']) else: - return _url_for_thread_comments(params["thread_id"]) + return _url_for_thread_comments(params['thread_id']) @classmethod def url(cls, action, params=None): if params is None: params = {} - if action in ["post"]: + if action in ['post']: return cls.url_for_comments(params) else: return super().url(action, params) def flagAbuse(self, user, voteable, course_id=None): - if voteable.type != "comment": + if voteable.type != 'comment': raise CommentClientRequestError("Can only flag comments") course_key = get_course_key(self.attributes.get("course_id") or course_id) @@ -118,7 +84,7 @@ def flagAbuse(self, user, voteable, course_id=None): voteable._update_from_response(response) def unFlagAbuse(self, user, voteable, removeAll, course_id=None): - if voteable.type != "comment": + if voteable.type != 'comment': raise CommentClientRequestError("Can only unflag comments") course_key = get_course_key(self.attributes.get("course_id") or course_id) @@ -136,7 +102,7 @@ def body_text(self): """ Return the text content of the comment html body. """ - soup = BeautifulSoup(self.body, "html.parser") + soup = BeautifulSoup(self.body, 'html.parser') return soup.get_text() @classmethod @@ -148,15 +114,12 @@ def get_user_comment_count(cls, user_id, course_ids): query_params = { "course_id": {"$in": course_ids}, "author_id": str(user_id), - "is_deleted": {"$ne": True}, - "_type": "Comment", + "_type": "Comment" } - return ForumComment()._collection.count_documents( - query_params - ) # pylint: disable=protected-access + return ForumComment()._collection.count_documents(query_params) # pylint: disable=protected-access @classmethod - def delete_user_comments(cls, user_id, course_ids, deleted_by=None): + def delete_user_comments(cls, user_id, course_ids): """ Deletes comments and responses of user in the given course_ids. TODO: Add support for MySQL backend as well @@ -165,66 +128,21 @@ def delete_user_comments(cls, user_id, course_ids, deleted_by=None): query_params = { "course_id": {"$in": course_ids}, "author_id": str(user_id), - "is_deleted": {"$ne": True}, } comments_deleted = 0 comments = ForumComment().get_list(**query_params) - log.info( - f"<> Fetched comments for user {user_id} in {time.time() - start_time} seconds" - ) + log.info(f"<> Fetched comments for user {user_id} in {time.time() - start_time} seconds") for comment in comments: start_time = time.time() comment_id = comment.get("_id") course_id = comment.get("course_id") if comment_id: - # Use forum_api.delete_comment which supports deleted_by parameter - forum_api.delete_comment( # pylint: disable=unexpected-keyword-arg - comment_id, course_id=course_id, deleted_by=deleted_by - ) + forum_api.delete_comment(comment_id, course_id=course_id) comments_deleted += 1 - log.info( - f"<> Deleted comment {comment_id} in {time.time() - start_time} seconds." - f" Comment Found: {comment_id is not None}" - ) + log.info(f"<> Deleted comment {comment_id} in {time.time() - start_time} seconds." + f" Comment Found: {comment_id is not None}") return comments_deleted - @classmethod - def get_user_deleted_comment_count(cls, user_id, course_ids): - """ - Returns count of deleted comments for user in the given course_ids. - """ - query_params = { - "course_id": {"$in": course_ids}, - "author_id": str(user_id), - "_type": "Comment", - "is_deleted": True, - } - return ForumComment()._collection.count_documents( - query_params - ) # pylint: disable=protected-access - - @classmethod - def restore_user_deleted_comments(cls, user_id, course_ids, restored_by=None): - """ - Restores (undeletes) comments of user in the given course_ids by setting is_deleted=False. - """ - return forum_api.restore_user_deleted_comments( - user_id=str(user_id), - course_ids=course_ids, - course_id=course_ids[0] if course_ids else None, - restored_by=restored_by, - ) - - @classmethod - def restore_comment(cls, comment_id, course_id=None, restored_by=None): - """ - Restores an individual soft-deleted comment by setting is_deleted=False - Public method for individual comment restoration - """ - return forum_api.restore_comment( - comment_id=comment_id, course_id=course_id, restored_by=restored_by - ) - def _url_for_thread_comments(thread_id): return f"{settings.PREFIX}/threads/{thread_id}/comments" diff --git a/openedx/core/djangoapps/django_comment_common/comment_client/models.py b/openedx/core/djangoapps/django_comment_common/comment_client/models.py index ddfcc37cc524..4544a463ed80 100644 --- a/openedx/core/djangoapps/django_comment_common/comment_client/models.py +++ b/openedx/core/djangoapps/django_comment_common/comment_client/models.py @@ -4,28 +4,24 @@ import logging import typing as t +from .utils import CommentClientRequestError, extract, perform_request, get_course_key from forum import api as forum_api -from openedx.core.djangoapps.discussions.config.waffle import ( - is_forum_v2_disabled_globally, - is_forum_v2_enabled, -) - -from .utils import CommentClientRequestError, extract, get_course_key, perform_request +from openedx.core.djangoapps.discussions.config.waffle import is_forum_v2_enabled, is_forum_v2_disabled_globally log = logging.getLogger(__name__) class Model: - accessible_fields = ["id"] - updatable_fields = ["id"] - initializable_fields = ["id"] + accessible_fields = ['id'] + updatable_fields = ['id'] + initializable_fields = ['id'] base_url = None default_retrieve_params = {} metric_tag_fields = [] - DEFAULT_ACTIONS_WITH_ID = ["get", "put", "delete"] - DEFAULT_ACTIONS_WITHOUT_ID = ["get_all", "post"] + DEFAULT_ACTIONS_WITH_ID = ['get', 'put', 'delete'] + DEFAULT_ACTIONS_WITHOUT_ID = ['get_all', 'post'] DEFAULT_ACTIONS = DEFAULT_ACTIONS_WITH_ID + DEFAULT_ACTIONS_WITHOUT_ID def __init__(self, *args, **kwargs): @@ -33,21 +29,18 @@ def __init__(self, *args, **kwargs): self.retrieved = False def __getattr__(self, name): - if name == "id": - return self.attributes.get("id", None) + if name == 'id': + return self.attributes.get('id', None) try: return self.attributes[name] - except KeyError as e: + except KeyError: if self.retrieved or self.id is None: - raise AttributeError(f"Field {name} does not exist") from e + raise AttributeError(f"Field {name} does not exist") # lint-amnesty, pylint: disable=raise-missing-from self.retrieve() return self.__getattr__(name) def __setattr__(self, name, value): - if ( - name == "attributes" - or name not in self.accessible_fields + self.updatable_fields - ): + if name == 'attributes' or name not in self.accessible_fields + self.updatable_fields: super().__setattr__(name, value) else: self.attributes[name] = value @@ -83,9 +76,7 @@ def _retrieve(self, *args, **kwargs): if not course_id: _, course_id = is_forum_v2_enabled_for_comment(self.id) if self.type == "comment": - response = forum_api.get_parent_comment( - comment_id=self.attributes["id"], course_id=course_id - ) + response = forum_api.get_parent_comment(comment_id=self.attributes["id"], course_id=course_id) else: raise CommentClientRequestError("Forum v2 API call is missing") self._update_from_response(response) @@ -100,11 +91,11 @@ def _metric_tags(self): record the class name of the model. """ tags = [ - f"{self.__class__.__name__}.{attr}:{self[attr]}" + f'{self.__class__.__name__}.{attr}:{self[attr]}' for attr in self.metric_tag_fields if attr in self.attributes ] - tags.append(f"model_class:{self.__class__.__name__}") + tags.append(f'model_class:{self.__class__.__name__}') return tags @classmethod @@ -123,11 +114,11 @@ def retrieve_all(cls, params=None): The parsed JSON response from the backend. """ return perform_request( - "get", - cls.url(action="get_all"), + 'get', + cls.url(action='get_all'), params, - metric_tags=[f"model_class:{cls.__name__}"], - metric_action="model.retrieve_all", + metric_tags=[f'model_class:{cls.__name__}'], + metric_action='model.retrieve_all', ) def _update_from_response(self, response_data): @@ -137,7 +128,8 @@ def _update_from_response(self, response_data): else: log.warning( "Unexpected field {field_name} in model {model_name}".format( - field_name=k, model_name=self.__class__.__name__ + field_name=k, + model_name=self.__class__.__name__ ) ) @@ -160,7 +152,7 @@ def save(self, params=None): Invokes Forum's POST/PUT service to create/update thread """ self.before_save(self) - if self.id: # if we have id already, treat this as an update + if self.id: # if we have id already, treat this as an update response = self.handle_update(params) else: # otherwise, treat this as an insert response = self.handle_create(params) @@ -168,25 +160,13 @@ def save(self, params=None): self._update_from_response(response) self.after_save(self) - def delete(self, course_id=None, deleted_by=None): + def delete(self, course_id=None): course_key = get_course_key(self.attributes.get("course_id") or course_id) response = None if self.type == "comment": - response = ( - forum_api.delete_comment( # pylint: disable=unexpected-keyword-arg - comment_id=self.attributes["id"], - course_id=str(course_key), - deleted_by=deleted_by, - ) - ) + response = forum_api.delete_comment(comment_id=self.attributes["id"], course_id=str(course_key)) elif self.type == "thread": - response = ( - forum_api.delete_thread( # pylint: disable=unexpected-keyword-arg - thread_id=self.attributes["id"], - course_id=str(course_key), - deleted_by=deleted_by, - ) - ) + response = forum_api.delete_thread(thread_id=self.attributes["id"], course_id=str(course_key)) if response is None: raise CommentClientRequestError("Forum v2 API call is missing") self.retrieved = True @@ -196,7 +176,7 @@ def delete(self, course_id=None, deleted_by=None): def url_with_id(cls, params=None): if params is None: params = {} - return cls.base_url + "/" + str(params["id"]) + return cls.base_url + '/' + str(params['id']) @classmethod def url_without_id(cls, params=None): @@ -207,21 +187,17 @@ def url(cls, action, params=None): if params is None: params = {} if cls.base_url is None: - raise CommentClientRequestError( - "Must provide base_url when using default url function" - ) - if action not in cls.DEFAULT_ACTIONS: + raise CommentClientRequestError("Must provide base_url when using default url function") + if action not in cls.DEFAULT_ACTIONS: # lint-amnesty, pylint: disable=no-else-raise raise ValueError( f"Invalid action {action}. The supported action must be in {str(cls.DEFAULT_ACTIONS)}" ) - if action in cls.DEFAULT_ACTIONS_WITH_ID: + elif action in cls.DEFAULT_ACTIONS_WITH_ID: try: return cls.url_with_id(params) - except KeyError as e: - raise CommentClientRequestError( - f"Cannot perform action {action} without id" - ) from e - else: # action must be in DEFAULT_ACTIONS_WITHOUT_ID now + except KeyError: + raise CommentClientRequestError(f"Cannot perform action {action} without id") # lint-amnesty, pylint: disable=raise-missing-from + else: # action must be in DEFAULT_ACTIONS_WITHOUT_ID now return cls.url_without_id() def handle_update(self, params=None): @@ -330,8 +306,8 @@ def handle_create(self, params=None): try: return handlers[self.type](course_key) - except KeyError as e: - raise CommentClientRequestError(f"Unsupported type: {self.type}") from e + except KeyError as exc: + raise CommentClientRequestError(f"Unsupported type: {self.type}") from exc def handle_create_comment(self, course_id): request_data = self.initializable_attributes() @@ -343,8 +319,8 @@ def handle_create_comment(self, course_id): "anonymous": request_data.get("anonymous", False), "anonymous_to_peers": request_data.get("anonymous_to_peers", False), } - if "endorsed" in request_data: - params["endorsed"] = request_data["endorsed"] + if 'endorsed' in request_data: + params['endorsed'] = request_data['endorsed'] if parent_id := self.attributes.get("parent_id"): params["parent_comment_id"] = parent_id response = forum_api.create_child_comment(**params) diff --git a/openedx/core/djangoapps/django_comment_common/comment_client/thread.py b/openedx/core/djangoapps/django_comment_common/comment_client/thread.py index 754fe0065f00..34ccd7bf2ce6 100644 --- a/openedx/core/djangoapps/django_comment_common/comment_client/thread.py +++ b/openedx/core/djangoapps/django_comment_common/comment_client/thread.py @@ -5,104 +5,50 @@ import time import typing as t -from django.core.exceptions import ObjectDoesNotExist from eventtracking import tracker -from rest_framework.serializers import ValidationError +from django.core.exceptions import ObjectDoesNotExist from forum import api as forum_api -from forum.api.threads import ( - prepare_thread_api_response, -) # pylint: disable=import-error -from forum.backend import get_backend # pylint: disable=import-error -from forum.backends.mongodb.threads import CommentThread # pylint: disable=import-error -from forum.utils import ForumV2RequestError # pylint: disable=import-error -from openedx.core.djangoapps.discussions.config.waffle import ( - is_forum_v2_disabled_globally, - is_forum_v2_enabled, -) +from forum.api.threads import prepare_thread_api_response +from forum.backend import get_backend +from forum.backends.mongodb.threads import CommentThread +from forum.utils import ForumV2RequestError +from rest_framework.serializers import ValidationError +from openedx.core.djangoapps.discussions.config.waffle import is_forum_v2_enabled, is_forum_v2_disabled_globally from . import models, settings, utils + log = logging.getLogger(__name__) class Thread(models.Model): # accessible_fields can be set and retrieved on the model accessible_fields = [ - "id", - "title", - "body", - "anonymous", - "anonymous_to_peers", - "course_id", - "closed", - "tags", - "votes", - "commentable_id", - "username", - "user_id", - "created_at", - "updated_at", - "comments_count", - "unread_comments_count", - "at_position_list", - "children", - "type", - "highlighted_title", - "highlighted_body", - "endorsed", - "read", - "group_id", - "group_name", - "pinned", - "abuse_flaggers", - "resp_skip", - "resp_limit", - "resp_total", - "thread_type", - "endorsed_responses", - "non_endorsed_responses", - "non_endorsed_resp_total", - "context", - "last_activity_at", - "closed_by", - "close_reason_code", - "edit_history", - "is_spam", - "ai_moderation_reason", - "abuse_flagged", - "is_deleted", - "deleted_at", - "deleted_by", + 'id', 'title', 'body', 'anonymous', 'anonymous_to_peers', 'course_id', + 'closed', 'tags', 'votes', 'commentable_id', 'username', 'user_id', + 'created_at', 'updated_at', 'comments_count', 'unread_comments_count', + 'at_position_list', 'children', 'type', 'highlighted_title', + 'highlighted_body', 'endorsed', 'read', 'group_id', 'group_name', 'pinned', + 'abuse_flaggers', 'resp_skip', 'resp_limit', 'resp_total', 'thread_type', + 'endorsed_responses', 'non_endorsed_responses', 'non_endorsed_resp_total', + 'context', 'last_activity_at', 'closed_by', 'close_reason_code', 'edit_history', + 'is_spam', 'ai_moderation_reason', 'abuse_flagged', ] # updateable_fields are sent in PUT requests updatable_fields = [ - "title", - "body", - "anonymous", - "anonymous_to_peers", - "course_id", - "read", - "closed", - "user_id", - "commentable_id", - "group_id", - "group_name", - "pinned", - "thread_type", - "close_reason_code", - "edit_reason_code", - "closing_user_id", - "editing_user_id", + 'title', 'body', 'anonymous', 'anonymous_to_peers', 'course_id', 'read', + 'closed', 'user_id', 'commentable_id', 'group_id', 'group_name', 'pinned', 'thread_type', + 'close_reason_code', 'edit_reason_code', 'closing_user_id', 'editing_user_id', ] # initializable_fields are sent in POST requests - initializable_fields = updatable_fields + ["thread_type", "context"] + initializable_fields = updatable_fields + ['thread_type', 'context'] base_url = f"{settings.PREFIX}/threads" - default_retrieve_params = {"recursive": False} - type = "thread" + default_retrieve_params = {'recursive': False} + type = 'thread' @classmethod def search(cls, query_params): @@ -112,83 +58,82 @@ def search(cls, query_params): # with_responses=False internally in the comment service, so no additional # optimization is required. params = { - "page": 1, - "per_page": 20, - "course_id": query_params["course_id"], + 'page': 1, + 'per_page': 20, + 'course_id': query_params['course_id'], } - params.update(utils.strip_blank(utils.strip_none(query_params))) + params.update( + utils.strip_blank(utils.strip_none(query_params)) + ) # Convert user_id and author_id to strings if present - for field in ["user_id", "author_id"]: + for field in ['user_id', 'author_id']: if value := params.get(field): params[field] = str(value) # Handle commentable_ids/commentable_id conversion - if commentable_ids := params.get("commentable_ids"): - params["commentable_ids"] = commentable_ids.split(",") - elif commentable_id := params.get("commentable_id"): - params["commentable_ids"] = [commentable_id] - params.pop("commentable_id", None) - if query_params.get("show_deleted", False): - params["is_deleted"] = True + if commentable_ids := params.get('commentable_ids'): + params['commentable_ids'] = commentable_ids.split(',') + elif commentable_id := params.get('commentable_id'): + params['commentable_ids'] = [commentable_id] + params.pop('commentable_id', None) + params = utils.clean_forum_params(params) - if query_params.get("text"): # Handle group_ids/group_id conversion - if group_ids := params.get("group_ids"): - params["group_ids"] = [ - int(group_id) for group_id in group_ids.split(",") - ] - elif group_id := params.get("group_id"): - params["group_ids"] = [int(group_id)] - params.pop("group_id", None) + if query_params.get('text'): # Handle group_ids/group_id conversion + if group_ids := params.get('group_ids'): + params['group_ids'] = [int(group_id) for group_id in group_ids.split(',')] + elif group_id := params.get('group_id'): + params['group_ids'] = [int(group_id)] + params.pop('group_id', None) response = forum_api.search_threads(**params) else: response = forum_api.get_user_threads(**params) - if query_params.get("text"): - search_query = query_params["text"] - course_id = query_params["course_id"] - group_id = query_params["group_id"] if "group_id" in query_params else None - requested_page = params["page"] - total_results = response.get("total_results") - corrected_text = response.get("corrected_text") + if query_params.get('text'): + search_query = query_params['text'] + course_id = query_params['course_id'] + group_id = query_params['group_id'] if 'group_id' in query_params else None + requested_page = params['page'] + total_results = response.get('total_results') + corrected_text = response.get('corrected_text') # Record search result metric to allow search quality analysis. # course_id is already included in the context for the event tracker tracker.emit( - "edx.forum.searched", + 'edx.forum.searched', { - "query": search_query, - "search_type": "Content", - "corrected_text": corrected_text, - "group_id": group_id, - "page": requested_page, - "total_results": total_results, - }, + 'query': search_query, + 'search_type': 'Content', + 'corrected_text': corrected_text, + 'group_id': group_id, + 'page': requested_page, + 'total_results': total_results, + } ) log.info( 'forum_text_search query="{search_query}" corrected_text="{corrected_text}" course_id={course_id} ' - "group_id={group_id} page={requested_page} total_results={total_results}".format( + 'group_id={group_id} page={requested_page} total_results={total_results}'.format( search_query=search_query, corrected_text=corrected_text, course_id=course_id, group_id=group_id, requested_page=requested_page, - total_results=total_results, + total_results=total_results ) ) return utils.CommentClientPaginatedResult( - collection=response.get("collection", []), - page=response.get("page", 1), - num_pages=response.get("num_pages", 1), - thread_count=response.get("thread_count", 0), - corrected_text=response.get("corrected_text", None), + collection=response.get('collection', []), + page=response.get('page', 1), + num_pages=response.get('num_pages', 1), + thread_count=response.get('thread_count', 0), + corrected_text=response.get('corrected_text', None) ) @classmethod def url_for_threads(cls, params=None): - if params and params.get("commentable_id"): + if params and params.get('commentable_id'): return "{prefix}/{commentable_id}/threads".format( prefix=settings.PREFIX, - commentable_id=params["commentable_id"], + commentable_id=params['commentable_id'], ) else: return f"{settings.PREFIX}/threads" @@ -201,9 +146,9 @@ def url_for_search_threads(cls): def url(cls, action, params=None): if params is None: params = {} - if action in ["get_all", "post"]: + if action in ['get_all', 'post']: return cls.url_for_threads(params) - elif action == "search": + elif action == 'search': return cls.url_for_search_threads() else: return super().url(action, params) @@ -213,23 +158,21 @@ def url(cls, action, params=None): # that subclasses don't need to override for this. def _retrieve(self, *args, **kwargs): request_params = { - "recursive": kwargs.get("recursive"), - "with_responses": kwargs.get("with_responses", False), - "user_id": kwargs.get("user_id"), - "mark_as_read": kwargs.get("mark_as_read", True), - "resp_skip": kwargs.get("response_skip"), - "resp_limit": kwargs.get("response_limit"), - "reverse_order": kwargs.get("reverse_order", False), - "merge_question_type_responses": kwargs.get( - "merge_question_type_responses", False - ), + 'recursive': kwargs.get('recursive'), + 'with_responses': kwargs.get('with_responses', False), + 'user_id': kwargs.get('user_id'), + 'mark_as_read': kwargs.get('mark_as_read', True), + 'resp_skip': kwargs.get('response_skip'), + 'resp_limit': kwargs.get('response_limit'), + 'reverse_order': kwargs.get('reverse_order', False), + 'merge_question_type_responses': kwargs.get('merge_question_type_responses', False) } request_params = utils.clean_forum_params(request_params) course_id = kwargs.get("course_id") if not course_id: _, course_id = is_forum_v2_enabled_for_thread(self.id) - if user_id := request_params.get("user_id"): - request_params["user_id"] = str(user_id) + if user_id := request_params.get('user_id'): + request_params['user_id'] = str(user_id) response = forum_api.get_thread( thread_id=self.id, params=request_params, @@ -238,7 +181,7 @@ def _retrieve(self, *args, **kwargs): self._update_from_response(response) def flagAbuse(self, user, voteable, course_id=None): - if voteable.type != "thread": + if voteable.type != 'thread': raise utils.CommentClientRequestError("Can only flag threads") course_key = utils.get_course_key(self.attributes.get("course_id") or course_id) @@ -246,12 +189,12 @@ def flagAbuse(self, user, voteable, course_id=None): thread_id=voteable.id, action="flag", user_id=str(user.id), - course_id=str(course_key), + course_id=str(course_key) ) voteable._update_from_response(response) def unFlagAbuse(self, user, voteable, removeAll, course_id=None): - if voteable.type != "thread": + if voteable.type != 'thread': raise utils.CommentClientRequestError("Can only unflag threads") course_key = utils.get_course_key(self.attributes.get("course_id") or course_id) @@ -260,7 +203,7 @@ def unFlagAbuse(self, user, voteable, removeAll, course_id=None): action="unflag", user_id=user.id, update_all=bool(removeAll), - course_id=str(course_key), + course_id=str(course_key) ) voteable._update_from_response(response) @@ -268,14 +211,18 @@ def unFlagAbuse(self, user, voteable, removeAll, course_id=None): def pin(self, user, thread_id, course_id=None): course_key = utils.get_course_key(self.attributes.get("course_id") or course_id) response = forum_api.pin_thread( - user_id=user.id, thread_id=thread_id, course_id=str(course_key) + user_id=user.id, + thread_id=thread_id, + course_id=str(course_key) ) self._update_from_response(response) def un_pin(self, user, thread_id, course_id=None): course_key = utils.get_course_key(self.attributes.get("course_id") or course_id) response = forum_api.unpin_thread( - user_id=user.id, thread_id=thread_id, course_id=str(course_key) + user_id=user.id, + thread_id=thread_id, + course_id=str(course_key) ) self._update_from_response(response) @@ -288,15 +235,12 @@ def get_user_threads_count(cls, user_id, course_ids): query_params = { "course_id": {"$in": course_ids}, "author_id": str(user_id), - "is_deleted": {"$ne": True}, - "_type": "CommentThread", + "_type": "CommentThread" } - return CommentThread()._collection.count_documents( - query_params - ) # pylint: disable=protected-access + return CommentThread()._collection.count_documents(query_params) # pylint: disable=protected-access @classmethod - def _delete_thread(cls, thread_id, course_id=None, deleted_by=None): + def _delete_thread(cls, thread_id, course_id=None): """ Deletes a thread """ @@ -313,53 +257,34 @@ def _delete_thread(cls, thread_id, course_id=None, deleted_by=None): ) from exc start_time = time.perf_counter() - # backend.delete_comments_of_a_thread(thread_id) - count_of_response_deleted, count_of_replies_deleted = ( - backend.soft_delete_comments_of_a_thread(thread_id, deleted_by) - ) - log.info( - f"{prefix} Delete comments of thread {time.perf_counter() - start_time} sec" - ) + backend.delete_comments_of_a_thread(thread_id) + log.info(f"{prefix} Delete comments of thread {time.perf_counter() - start_time} sec") try: start_time = time.perf_counter() serialized_data = prepare_thread_api_response(thread, backend) - log.info( - f"{prefix} Prepare response {time.perf_counter() - start_time} sec" - ) + log.info(f"{prefix} Prepare response {time.perf_counter() - start_time} sec") except ValidationError as error: log.error(f"Validation error in get_thread: {error}") - raise ForumV2RequestError( - "Failed to prepare thread API response" - ) from error + raise ForumV2RequestError("Failed to prepare thread API response") from error start_time = time.perf_counter() backend.delete_subscriptions_of_a_thread(thread_id) - log.info( - f"{prefix} Delete subscriptions {time.perf_counter() - start_time} sec" - ) + log.info(f"{prefix} Delete subscriptions {time.perf_counter() - start_time} sec") start_time = time.perf_counter() - # result = backend.delete_thread(thread_id) - result = backend.soft_delete_thread(thread_id, deleted_by) + result = backend.delete_thread(thread_id) log.info(f"{prefix} Delete thread {time.perf_counter() - start_time} sec") if result and not (thread["anonymous"] or thread["anonymous_to_peers"]): start_time = time.perf_counter() backend.update_stats_for_course( - thread["author_id"], - thread["course_id"], - threads=-1, - responses=-count_of_response_deleted, - replies=-count_of_replies_deleted, - deleted_threads=1, - deleted_responses=count_of_response_deleted, - deleted_replies=count_of_replies_deleted, + thread["author_id"], thread["course_id"], threads=-1 ) log.info(f"{prefix} Update stats {time.perf_counter() - start_time} sec") return serialized_data @classmethod - def delete_user_threads(cls, user_id, course_ids, deleted_by=None): + def delete_user_threads(cls, user_id, course_ids): """ Deletes threads of user in the given course_ids. TODO: Add support for MySQL backend as well @@ -368,65 +293,21 @@ def delete_user_threads(cls, user_id, course_ids, deleted_by=None): query_params = { "course_id": {"$in": course_ids}, "author_id": str(user_id), - "is_deleted": {"$ne": True}, } threads_deleted = 0 threads = CommentThread().get_list(**query_params) - log.info( - f"<> Fetched threads for user {user_id} in {time.time() - start_time} seconds" - ) + log.info(f"<> Fetched threads for user {user_id} in {time.time() - start_time} seconds") for thread in threads: start_time = time.time() thread_id = thread.get("_id") course_id = thread.get("course_id") if thread_id: - cls._delete_thread( - thread_id, course_id=course_id, deleted_by=deleted_by - ) + cls._delete_thread(thread_id, course_id=course_id) threads_deleted += 1 - log.info( - f"<> Deleted thread {thread_id} in {time.time() - start_time} seconds." - f" Thread Found: {thread_id is not None}" - ) + log.info(f"<> Deleted thread {thread_id} in {time.time() - start_time} seconds." + f" Thread Found: {thread_id is not None}") return threads_deleted - @classmethod - def get_user_deleted_threads_count(cls, user_id, course_ids): - """ - Returns count of deleted threads for user in the given course_ids. - """ - query_params = { - "course_id": {"$in": course_ids}, - "author_id": str(user_id), - "_type": "CommentThread", - "is_deleted": True, - } - return CommentThread()._collection.count_documents( - query_params - ) # pylint: disable=protected-access - - @classmethod - def restore_user_deleted_threads(cls, user_id, course_ids, restored_by=None): - """ - Restores (undeletes) threads of user in the given course_ids by setting is_deleted=False. - """ - return forum_api.restore_user_deleted_threads( - user_id=str(user_id), - course_ids=course_ids, - course_id=course_ids[0] if course_ids else None, - restored_by=restored_by, - ) - - @classmethod - def restore_thread(cls, thread_id, course_id=None, restored_by=None): - """ - Restores an individual soft-deleted thread by setting is_deleted=False - Public method for individual thread restoration - """ - return forum_api.restore_thread( - thread_id=thread_id, course_id=course_id, restored_by=restored_by - ) - def _url_for_flag_abuse_thread(thread_id): return f"{settings.PREFIX}/threads/{thread_id}/abuse_flag" From a0133462cee96cd7c608485f6f79f4b969dc552b Mon Sep 17 00:00:00 2001 From: Akanshu Aich Date: Wed, 7 Jan 2026 21:48:35 +0530 Subject: [PATCH 164/351] fix: improve logging for user retirement errors with exception details (#76) Ticket: https://2u-internal.atlassian.net/browse/BOMS-290 --- openedx/core/djangoapps/user_api/accounts/views.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/openedx/core/djangoapps/user_api/accounts/views.py b/openedx/core/djangoapps/user_api/accounts/views.py index f338fd510177..5180c1adb0ec 100644 --- a/openedx/core/djangoapps/user_api/accounts/views.py +++ b/openedx/core/djangoapps/user_api/accounts/views.py @@ -1108,7 +1108,7 @@ def post(self, request): user=retirement.user, ) except UserRetirementStatus.DoesNotExist: - log.error( + log.exception( 'UserRetirementStatus not found for retirement action' ) record_exception() @@ -1118,7 +1118,7 @@ def post(self, request): user_id = retirement.user.id except AttributeError: user_id = 'unknown' - log.error( + log.exception( 'RetirementStateError during user retirement: user_id=%s, error=%s', user_id, str(exc) ) @@ -1129,9 +1129,10 @@ def post(self, request): user_id = retirement.user.id except AttributeError: user_id = 'unknown' - log.error( - 'Unexpected error during user retirement: user_id=%s, error=%s', - user_id, str(exc) + exception_type = type(exc).__name__ + log.exception( + 'Unexpected error during user retirement: user_id=%s, exception_type=%s, error=%s', + user_id, exception_type, str(exc) ) record_exception() return Response(str(exc), status=status.HTTP_500_INTERNAL_SERVER_ERROR) From 597a2a0cc3e9937c26de43411bf4b117bb03c57f Mon Sep 17 00:00:00 2001 From: Devasia Joseph Date: Thu, 8 Jan 2026 19:11:50 +0530 Subject: [PATCH 165/351] fix: improve beta label contrast in gamesxblock (#80) * fix: improve beta label contrast in gamesxblock * fix: address copilot comment Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --------- Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- cms/static/sass/elements/_modules.scss | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cms/static/sass/elements/_modules.scss b/cms/static/sass/elements/_modules.scss index 1e9f52691fc8..f1c871ba1d95 100644 --- a/cms/static/sass/elements/_modules.scss +++ b/cms/static/sass/elements/_modules.scss @@ -163,8 +163,8 @@ display: inline-block; color: $uxpl-green-base; - background-color: theme-color("inverse"); - border-color: theme-color("inverse"); + background-color: $white; + border-color: $white; border-radius: 3px; font-size: 90%; From 5d33a7941f1ed7397ee3ee876850d4c19e563fc7 Mon Sep 17 00:00:00 2001 From: Muhammad Arslan Date: Thu, 8 Jan 2026 20:33:04 +0500 Subject: [PATCH 166/351] fix: race condition in shared runtime services (#37825) There is a singleton SplitMongoModuleStore instance that is returned whenever we call the ubiquitous modulestore() function (wrapped in a MixedModuleStore). During initialization, SplitMongoModuleStore sets up a small handful of XBlock runtime services that are intended to be shared globally: i18n, fs, cache. When we get an individual block back from the store using get_item(), SplitMongoModuleStore creates a SplitModuleStoreRuntime using SplitMongoModuleStore.create_runtime(). These runtimes are intended to be modified on a per-item, and later per-user basis (using prepare_runtime_for_user()). Prior to this commit, the create_runtime() method was assigning the globally shared SplitMongoModuleStore.services dict directly to the newly instantiated SplitModuleStoreRuntime. This meant that even though each block had its own _services dict, they were all in fact pointing to the same underlying object. This exposed us to a risk of multiple threads contaminating each other's SplitModuleStoreRuntime services when deployed under load in multithreaded mode. We believe this led to a race condition that caused student submissions to be mis-scored in some cases. This commit makes a copy of the SplitMongoModuleStore.services dict for each SplitModuleStoreRuntime. The baseline global services are still shared, but other per-item and per-user services are now better isolated from each other. This commit also includes a small modification to the PartitionService, which up until this point had relied on the (incorrect) shared instance behavior. The details are provided in the comments in the PartitionService __init__(). It's worth noting that the historical rationale for having a singleton ModuleStore instance is that the ModuleStore used to be extremely expensive to initialize. This was because at one point, the init process required reading entire XML-based courses into memory, or pre-computing complex field inheritance caches. This is no longer the case, and SplitMongoModuleStore initialization is in the 1-2 ms range, with most of that being for PyMongo's connection setup. We should try to fully remove the global singleton in the Verawood release cycle in order to make this kind of bug less likely. --- xmodule/modulestore/split_mongo/split.py | 6 +++- xmodule/partitions/partitions_service.py | 43 ++++++++++++++++++++++-- 2 files changed, 46 insertions(+), 3 deletions(-) diff --git a/xmodule/modulestore/split_mongo/split.py b/xmodule/modulestore/split_mongo/split.py index 07c985c21cc5..74539fabed90 100644 --- a/xmodule/modulestore/split_mongo/split.py +++ b/xmodule/modulestore/split_mongo/split.py @@ -3283,7 +3283,11 @@ def create_runtime(self, course_entry, lazy): """ Create the proper runtime for this course """ - services = self.services + # A single SplitMongoModuleStore may create many SplitModuleStoreRuntimes, + # each of which will later modify its internal dict of services on a per-item and often per-user basis. + # Therefore, it's critical that we make a new copy of our baseline services dict here, + # so that each runtime is free to add and replace its services without impacting other runtimes. + services = self.services.copy() # Only the CourseBlock can have user partitions. Therefore, creating the PartitionService with the library key # instead of the course key does not work. The XBlock validation in Studio fails with the following message: # "This component's access settings refer to deleted or invalid group configurations.". diff --git a/xmodule/partitions/partitions_service.py b/xmodule/partitions/partitions_service.py index 6cffd2c20c7b..ddd37d5212f5 100644 --- a/xmodule/partitions/partitions_service.py +++ b/xmodule/partitions/partitions_service.py @@ -99,8 +99,47 @@ class PartitionService: with a given course. """ - def __init__(self, course_id, cache=None, course=None): - self._course_id = course_id + def __init__(self, course_id: CourseKey, cache=None, course=None): + """Create a new ParititonService. This is user-specific.""" + + # There is a surprising amount of complexity in how to save the + # course_id we were passed in this constructor. + if course_id.org and course_id.course and course_id.run: + # This is the normal case, where we're instantiated with a CourseKey + # that has org, course, and run information. It will also often have + # a version_guid attached in this case, and we will want to strip + # that off in most cases. + # + # The reason for this is that the PartitionService is going to get + # recreated for every runtime (i.e. every block that's created for a + # user). Say you do the following: + # + # 1. You query the modulestore's get_item() for block A. + # 2. You update_item() for a different block B + # 3. You publish block B. + # + # When get_item() was called, a SplitModuleStoreRuntime was created + # for block A and it was given a CourseKey that had the version_guid + # encoded in it. If we persist that CourseKey with the version guid + # intact, then it will be incorrect after B is published, and any + # future access checks on A will break because it will try to query + # for a version of the course that is no longer published. + # + # Note that we still need to keep the branch information, or else + # this wouldn't work right in preview mode. + self._course_id = course_id.replace(version_guid=None) + else: + # If we're here, it means that the CourseKey we were sent doesn't + # have an org, course, and run. A much less common (but still legal) + # way to query by CourseKey involves a version_guid-only query, i.e. + # everything is None but the version_guid. In this scenario, it + # doesn't make sense to remove the one identifying piece of + # information we have, so we just assign the CourseKey without + # modification. We *could* potentially query the modulestore + # here and get the more normal form of the CourseKey, but that would + # be much more expensive and require database access. + self._course_id = course_id + self._cache = cache self.course = course From f2e024cc9e6a8550621afb0709c0badcfc2f97ab Mon Sep 17 00:00:00 2001 From: jajjibhai008 <86868918+jajjibhai008@users.noreply.github.com> Date: Fri, 9 Jan 2026 11:23:07 +0000 Subject: [PATCH 167/351] feat: Upgrade Python dependency edx-enterprise Commit generated by workflow `edx/edx-platform/.github/workflows/upgrade-one-python-dependency.yml@refs/heads/release-ulmo` --- requirements/common_constraints.txt | 7 ------- requirements/constraints.txt | 2 +- requirements/edx/base.txt | 16 ++++------------ requirements/edx/development.txt | 12 ++---------- requirements/edx/doc.txt | 11 ++--------- requirements/edx/testing.txt | 11 ++--------- requirements/pip.txt | 4 +--- 7 files changed, 12 insertions(+), 51 deletions(-) diff --git a/requirements/common_constraints.txt b/requirements/common_constraints.txt index 1f3e81f50334..748858b7015a 100644 --- a/requirements/common_constraints.txt +++ b/requirements/common_constraints.txt @@ -22,10 +22,3 @@ Django<6.0 # elastic search changelog: https://www.elastic.co/guide/en/enterprise-search/master/release-notes-7.14.0.html # See https://github.com/openedx/edx-platform/issues/35126 for more info elasticsearch<7.14.0 - -# pip 25.3 is incompatible with pip-tools hence causing failures during the build process -# Make upgrade command and all requirements upgrade jobs are broken due to this. -# See issue https://github.com/openedx/public-engineering/issues/440 for details regarding the ongoing fix. -# The constraint can be removed once a release (pip-tools > 7.5.1) is available with support for pip 25.3 -# Issue to track this dependency and unpin later on: https://github.com/openedx/edx-lint/issues/503 -pip<25.3 diff --git a/requirements/constraints.txt b/requirements/constraints.txt index e4ecb8e51cde..92b9d9a5c5ec 100644 --- a/requirements/constraints.txt +++ b/requirements/constraints.txt @@ -42,7 +42,7 @@ django-stubs<6 # The team that owns this package will manually bump this package rather than having it pulled in automatically. # This is to allow them to better control its deployment and to do it in a process that works better # for them. -edx-enterprise==6.5.7 +edx-enterprise==6.6.1 # Date: 2023-07-26 # Our legacy Sass code is incompatible with anything except this ancient libsass version. diff --git a/requirements/edx/base.txt b/requirements/edx/base.txt index ac4608f0eca1..42c7d55b301b 100644 --- a/requirements/edx/base.txt +++ b/requirements/edx/base.txt @@ -9,9 +9,7 @@ acid-xblock==0.4.1 aiohappyeyeballs==2.6.1 # via aiohttp aiohttp==3.13.0 - # via - # geoip2 - # openai + # via geoip2 aiosignal==1.4.0 # via aiohttp amqp==5.3.1 @@ -476,7 +474,7 @@ edx-drf-extensions==10.6.0 # edxval # enterprise-integrated-channels # openedx-learning -edx-enterprise==6.5.7 +edx-enterprise==6.6.1 # via # -c requirements/constraints.txt # -r requirements/edx/kernel.in @@ -807,13 +805,10 @@ oauthlib==3.3.1 # xblocks-contrib olxcleaner==0.3.0 # via -r requirements/edx/kernel.in -openai==0.28.1 - # via - # -c requirements/constraints.txt - # edx-enterprise openedx-atlas==0.7.0 # via # -r requirements/edx/kernel.in + # edx-enterprise # enterprise-integrated-channels # openedx-forum openedx-calc==4.0.2 @@ -1052,7 +1047,6 @@ requests==2.32.5 # google-cloud-storage # mailsnake # meilisearch - # openai # openedx-forum # optimizely-sdk # pylti1p3 @@ -1172,9 +1166,7 @@ tomlkit==0.13.3 # openedx-learning # snowflake-connector-python tqdm==4.67.1 - # via - # nltk - # openai + # via nltk typing-extensions==4.15.0 # via # aiosignal diff --git a/requirements/edx/development.txt b/requirements/edx/development.txt index f3ae39f02e4d..cddac634325a 100644 --- a/requirements/edx/development.txt +++ b/requirements/edx/development.txt @@ -22,7 +22,6 @@ aiohttp==3.13.0 # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt # geoip2 - # openai aiosignal==1.4.0 # via # -r requirements/edx/doc.txt @@ -749,7 +748,7 @@ edx-drf-extensions==10.6.0 # edxval # enterprise-integrated-channels # openedx-learning -edx-enterprise==6.5.7 +edx-enterprise==6.6.1 # via # -c requirements/constraints.txt # -r requirements/edx/doc.txt @@ -1345,16 +1344,11 @@ olxcleaner==0.3.0 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt -openai==0.28.1 - # via - # -c requirements/constraints.txt - # -r requirements/edx/doc.txt - # -r requirements/edx/testing.txt - # edx-enterprise openedx-atlas==0.7.0 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt + # edx-enterprise # enterprise-integrated-channels # openedx-forum openedx-calc==4.0.2 @@ -1826,7 +1820,6 @@ requests==2.32.5 # google-cloud-storage # mailsnake # meilisearch - # openai # openedx-forum # optimizely-sdk # pact-python @@ -2098,7 +2091,6 @@ tqdm==4.67.1 # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt # nltk - # openai types-pyyaml==6.0.12.20250915 # via # django-stubs diff --git a/requirements/edx/doc.txt b/requirements/edx/doc.txt index 4ffb83fdf8ca..1d5ef3e5d59a 100644 --- a/requirements/edx/doc.txt +++ b/requirements/edx/doc.txt @@ -16,7 +16,6 @@ aiohttp==3.13.0 # via # -r requirements/edx/base.txt # geoip2 - # openai aiosignal==1.4.0 # via # -r requirements/edx/base.txt @@ -559,7 +558,7 @@ edx-drf-extensions==10.6.0 # edxval # enterprise-integrated-channels # openedx-learning -edx-enterprise==6.5.7 +edx-enterprise==6.6.1 # via # -c requirements/constraints.txt # -r requirements/edx/base.txt @@ -979,14 +978,10 @@ oauthlib==3.3.1 # xblocks-contrib olxcleaner==0.3.0 # via -r requirements/edx/base.txt -openai==0.28.1 - # via - # -c requirements/constraints.txt - # -r requirements/edx/base.txt - # edx-enterprise openedx-atlas==0.7.0 # via # -r requirements/edx/base.txt + # edx-enterprise # enterprise-integrated-channels # openedx-forum openedx-calc==4.0.2 @@ -1282,7 +1277,6 @@ requests==2.32.5 # google-cloud-storage # mailsnake # meilisearch - # openai # openedx-forum # optimizely-sdk # pylti1p3 @@ -1485,7 +1479,6 @@ tqdm==4.67.1 # via # -r requirements/edx/base.txt # nltk - # openai typing-extensions==4.15.0 # via # -r requirements/edx/base.txt diff --git a/requirements/edx/testing.txt b/requirements/edx/testing.txt index c23542e20e49..ce7040f87793 100644 --- a/requirements/edx/testing.txt +++ b/requirements/edx/testing.txt @@ -14,7 +14,6 @@ aiohttp==3.13.0 # via # -r requirements/edx/base.txt # geoip2 - # openai aiosignal==1.4.0 # via # -r requirements/edx/base.txt @@ -580,7 +579,7 @@ edx-drf-extensions==10.6.0 # edxval # enterprise-integrated-channels # openedx-learning -edx-enterprise==6.5.7 +edx-enterprise==6.6.1 # via # -c requirements/constraints.txt # -r requirements/edx/base.txt @@ -1024,14 +1023,10 @@ oauthlib==3.3.1 # xblocks-contrib olxcleaner==0.3.0 # via -r requirements/edx/base.txt -openai==0.28.1 - # via - # -c requirements/constraints.txt - # -r requirements/edx/base.txt - # edx-enterprise openedx-atlas==0.7.0 # via # -r requirements/edx/base.txt + # edx-enterprise # enterprise-integrated-channels # openedx-forum openedx-calc==4.0.2 @@ -1393,7 +1388,6 @@ requests==2.32.5 # google-cloud-storage # mailsnake # meilisearch - # openai # openedx-forum # optimizely-sdk # pact-python @@ -1559,7 +1553,6 @@ tqdm==4.67.1 # via # -r requirements/edx/base.txt # nltk - # openai typing-extensions==4.15.0 # via # -r requirements/edx/base.txt diff --git a/requirements/pip.txt b/requirements/pip.txt index c6158d38e981..dec15874f740 100644 --- a/requirements/pip.txt +++ b/requirements/pip.txt @@ -9,8 +9,6 @@ wheel==0.45.1 # The following packages are considered to be unsafe in a requirements file: pip==25.2 - # via - # -c requirements/common_constraints.txt - # -r requirements/pip.in + # via -r requirements/pip.in setuptools==80.9.0 # via -r requirements/pip.in From c71aca8903a6109f1621b201cd3d8ad7017267b5 Mon Sep 17 00:00:00 2001 From: Vivek Date: Fri, 9 Jan 2026 17:21:05 +0530 Subject: [PATCH 168/351] feat: add new API to retrieve all components in a unit (#70) * feat: add new API to retrieve all components in a unit * feat: add waffle flag to enable unit expanded view * fix: test cases --- .../v1/serializers/course_waffle_flags.py | 9 ++ .../contentstore/rest_api/v1/urls.py | 6 + .../rest_api/v1/views/__init__.py | 7 +- .../views/tests/test_course_waffle_flags.py | 1 + .../rest_api/v1/views/unit_handler.py | 135 ++++++++++++++++++ cms/djangoapps/contentstore/toggles.py | 20 +++ 6 files changed, 173 insertions(+), 5 deletions(-) create mode 100644 cms/djangoapps/contentstore/rest_api/v1/views/unit_handler.py diff --git a/cms/djangoapps/contentstore/rest_api/v1/serializers/course_waffle_flags.py b/cms/djangoapps/contentstore/rest_api/v1/serializers/course_waffle_flags.py index 3efb7b6226d4..fde90163f803 100644 --- a/cms/djangoapps/contentstore/rest_api/v1/serializers/course_waffle_flags.py +++ b/cms/djangoapps/contentstore/rest_api/v1/serializers/course_waffle_flags.py @@ -11,6 +11,7 @@ class CourseWaffleFlagsSerializer(serializers.Serializer): """ Serializer for course waffle flags """ + use_new_home_page = serializers.SerializerMethodField() use_new_custom_pages = serializers.SerializerMethodField() use_new_schedule_details_page = serializers.SerializerMethodField() @@ -31,6 +32,7 @@ class CourseWaffleFlagsSerializer(serializers.Serializer): use_react_markdown_editor = serializers.SerializerMethodField() use_video_gallery_flow = serializers.SerializerMethodField() enable_course_optimizer_check_prev_run_links = serializers.SerializerMethodField() + enable_unit_expanded_view = serializers.SerializerMethodField() def get_course_key(self): """ @@ -175,3 +177,10 @@ def get_enable_course_optimizer_check_prev_run_links(self, obj): """ course_key = self.get_course_key() return toggles.enable_course_optimizer_check_prev_run_links(course_key) + + def get_enable_unit_expanded_view(self, obj): + """ + Method to get the enable_unit_expanded_view waffle flag + """ + course_key = self.get_course_key() + return toggles.enable_unit_expanded_view(course_key) diff --git a/cms/djangoapps/contentstore/rest_api/v1/urls.py b/cms/djangoapps/contentstore/rest_api/v1/urls.py index 685a81d778ce..8a94f0b0e040 100644 --- a/cms/djangoapps/contentstore/rest_api/v1/urls.py +++ b/cms/djangoapps/contentstore/rest_api/v1/urls.py @@ -25,6 +25,7 @@ HomePageView, ProctoredExamSettingsView, ProctoringErrorsView, + UnitComponentsView, VideoDownloadView, VideoUsageView, vertical_container_children_redirect_view, @@ -144,6 +145,11 @@ CourseWaffleFlagsView.as_view(), name="course_waffle_flags" ), + re_path( + fr'^unit_handler/{settings.USAGE_KEY_PATTERN}$', + UnitComponentsView.as_view(), + name="unit_components" + ), # Authoring API # Do not use under v1 yet (Nov. 23). The Authoring API is still experimental and the v0 versions should be used diff --git a/cms/djangoapps/contentstore/rest_api/v1/views/__init__.py b/cms/djangoapps/contentstore/rest_api/v1/views/__init__.py index d4fcfd5f2e3f..25c0157904e9 100644 --- a/cms/djangoapps/contentstore/rest_api/v1/views/__init__.py +++ b/cms/djangoapps/contentstore/rest_api/v1/views/__init__.py @@ -14,9 +14,6 @@ from .proctoring import ProctoredExamSettingsView, ProctoringErrorsView from .settings import CourseSettingsView from .textbooks import CourseTextbooksView +from .unit_handler import UnitComponentsView from .vertical_block import ContainerHandlerView, vertical_container_children_redirect_view -from .videos import ( - CourseVideosView, - VideoDownloadView, - VideoUsageView, -) +from .videos import CourseVideosView, VideoDownloadView, VideoUsageView diff --git a/cms/djangoapps/contentstore/rest_api/v1/views/tests/test_course_waffle_flags.py b/cms/djangoapps/contentstore/rest_api/v1/views/tests/test_course_waffle_flags.py index f45cc48810d6..a788ce4af3b5 100644 --- a/cms/djangoapps/contentstore/rest_api/v1/views/tests/test_course_waffle_flags.py +++ b/cms/djangoapps/contentstore/rest_api/v1/views/tests/test_course_waffle_flags.py @@ -38,6 +38,7 @@ class CourseWaffleFlagsViewTest(CourseTestCase): "use_react_markdown_editor": False, "use_video_gallery_flow": False, "enable_course_optimizer_check_prev_run_links": False, + "enable_unit_expanded_view": False, } def setUp(self): diff --git a/cms/djangoapps/contentstore/rest_api/v1/views/unit_handler.py b/cms/djangoapps/contentstore/rest_api/v1/views/unit_handler.py new file mode 100644 index 000000000000..18a2376b3922 --- /dev/null +++ b/cms/djangoapps/contentstore/rest_api/v1/views/unit_handler.py @@ -0,0 +1,135 @@ +"""API Views for unit components handler""" + +import logging + +import edx_api_doc_tools as apidocs +from django.http import HttpResponseBadRequest, HttpResponseForbidden +from opaque_keys.edx.keys import UsageKey +from rest_framework.request import Request +from rest_framework.response import Response +from rest_framework.views import APIView + +from cms.djangoapps.contentstore.rest_api.v1.mixins import ContainerHandlerMixin +from cms.djangoapps.contentstore.toggles import enable_unit_expanded_view +from openedx.core.lib.api.view_utils import view_auth_classes +from xmodule.modulestore.django import modulestore +from xmodule.modulestore.exceptions import ItemNotFoundError + +log = logging.getLogger(__name__) + + +@view_auth_classes(is_authenticated=True) +class UnitComponentsView(APIView, ContainerHandlerMixin): + """ + View to get all components in a unit by usage key. + """ + + @apidocs.schema( + parameters=[ + apidocs.string_parameter( + "usage_key_string", + apidocs.ParameterLocation.PATH, + description="Unit usage key", + ), + ], + responses={ + 200: "List of components in the unit", + 400: "Invalid usage key or unit not found.", + 401: "The requester is not authenticated.", + 404: "The requested unit does not exist.", + }, + ) + def get(self, request: Request, usage_key_string: str): + """ + Get all components in a unit. + + **Example Request** + + GET /api/contentstore/v1/unit_handler/{usage_key_string} + + **Response Values** + + If the request is successful, an HTTP 200 "OK" response is returned. + + The HTTP 200 response contains a dict with a list of all components + in the unit, including their display names, block types, and block IDs. + + **Example Response** + + ```json + { + "unit_id": "block-v1:edX+DemoX+Demo_Course+type@vertical+block@vertical_id", + "display_name": "My Unit", + "components": [ + { + "block_id": "block-v1:edX+DemoX+Demo_Course+type@video+block@video_id", + "block_type": "video", + "display_name": "Introduction Video" + }, + { + "block_id": "block-v1:edX+DemoX+Demo_Course+type@html+block@html_id", + "block_type": "html", + "display_name": "Text Content" + }, + { + "block_id": "block-v1:edX+DemoX+Demo_Course+type@problem+block@problem_id", + "block_type": "problem", + "display_name": "Practice Problem" + } + ] + } + ``` + """ + try: + usage_key = UsageKey.from_string(usage_key_string) + except Exception as e: # pylint: disable=broad-exception-caught + log.error(f"Invalid usage key: {usage_key_string}, error: {str(e)}") + return HttpResponseBadRequest("Invalid usage key format") + + try: + # Get the unit xblock + unit_xblock = modulestore().get_item(usage_key) + + # Verify it's a vertical (unit) + if unit_xblock.category != "vertical": + return HttpResponseBadRequest( + "The provided usage key is not a unit (vertical)" + ) + + if not enable_unit_expanded_view(unit_xblock.location.course_key): + return HttpResponseForbidden( + "Unit expanded view is disabled for this course" + ) + + components = [] + + # Get all children (components) of the unit + if unit_xblock.has_children: + for child_usage_key in unit_xblock.children: + try: + child_xblock = modulestore().get_item(child_usage_key) + components.append( + { + "block_id": str(child_xblock.location), + "block_type": child_xblock.category, + "display_name": child_xblock.display_name_with_default, + } + ) + except ItemNotFoundError: + log.warning(f"Child block not found: {child_usage_key}") + continue + + response_data = { + "unit_id": str(usage_key), + "display_name": unit_xblock.display_name_with_default, + "components": components, + } + + return Response(response_data) + + except ItemNotFoundError: + log.error(f"Unit not found: {usage_key_string}") + return HttpResponseBadRequest("Unit not found") + except Exception as e: # pylint: disable=broad-exception-caught + log.error(f"Error retrieving unit components: {str(e)}") + return HttpResponseBadRequest(f"Error retrieving unit components: {str(e)}") diff --git a/cms/djangoapps/contentstore/toggles.py b/cms/djangoapps/contentstore/toggles.py index c287f8c4dbec..c3dba4a6f4a4 100644 --- a/cms/djangoapps/contentstore/toggles.py +++ b/cms/djangoapps/contentstore/toggles.py @@ -682,3 +682,23 @@ def enable_course_optimizer_check_prev_run_links(course_key): Returns a boolean if previous run course optimizer feature is enabled for the given course. """ return ENABLE_COURSE_OPTIMIZER_CHECK_PREV_RUN_LINKS.is_enabled(course_key) + + +# .. toggle_name: contentstore.enable_unit_expanded_view +# .. toggle_implementation: CourseWaffleFlag +# .. toggle_default: False +# .. toggle_description: When enabled, the Unit Expanded View feature in the Course Outline is activated. +# .. toggle_use_cases: temporary +# .. toggle_creation_date: 2026-01-01 +# .. toggle_target_removal_date: 2026-06-01 +# .. toggle_tickets: TNL2-473 +ENABLE_UNIT_EXPANDED_VIEW = CourseWaffleFlag( + f"{CONTENTSTORE_NAMESPACE}.enable_unit_expanded_view", __name__ +) + + +def enable_unit_expanded_view(course_key): + """ + Returns a boolean if the Unit Expanded View feature is enabled for the given course. + """ + return ENABLE_UNIT_EXPANDED_VIEW.is_enabled(course_key) From afd869cfc0ba6e4419fec2aca9ee909e8aa3b2d3 Mon Sep 17 00:00:00 2001 From: Nathan Sprenkle Date: Fri, 9 Jan 2026 12:22:38 -0500 Subject: [PATCH 169/351] feat: add toggle for unifying site & translation language (#81) Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- lms/djangoapps/courseware/toggles.py | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/lms/djangoapps/courseware/toggles.py b/lms/djangoapps/courseware/toggles.py index f9f083cad42e..3ce78c554dd0 100644 --- a/lms/djangoapps/courseware/toggles.py +++ b/lms/djangoapps/courseware/toggles.py @@ -2,7 +2,7 @@ Toggles for courseware in-course experience. """ -from edx_toggles.toggles import SettingToggle, WaffleSwitch +from edx_toggles.toggles import SettingToggle, WaffleFlag, WaffleSwitch from openedx.core.djangoapps.waffle_utils import CourseWaffleFlag @@ -168,6 +168,19 @@ f'{WAFFLE_FLAG_NAMESPACE}.discovery_default_language_filter', __name__ ) +# .. toggle_name: courseware.unify_site_and_translation_language +# .. toggle_implementation: WaffleFlag +# .. toggle_default: False +# .. toggle_description: Update LMS to use site language for xpert unit translations and enable new header site language switcher. +# .. toggle_use_cases: opt_in +# .. toggle_creation_date: 2026-01-08 +# .. toggle_target_removal_date: None +# .. toggle_warning: None. +# .. toggle_tickets: https://github.com/edx/edx-platform/pull/81 +ENABLE_UNIFIED_SITE_AND_TRANSLATION_LANGUAGE = WaffleFlag( + f'{WAFFLE_FLAG_NAMESPACE}.unify_site_and_translation_language', __name__ +) + def course_exit_page_is_active(course_key): return COURSEWARE_MICROFRONTEND_COURSE_EXIT_PAGE.is_enabled(course_key) From 1c1e08ffe0e835cce7691bc6d23e82aa05220798 Mon Sep 17 00:00:00 2001 From: Akanshu Aich Date: Mon, 12 Jan 2026 21:14:30 +0530 Subject: [PATCH 170/351] fix: handle CourseOverview.DoesNotExist exception in optout creation (#77) Ticket: https://2u-internal.atlassian.net/browse/BOMS-370 --- lms/djangoapps/bulk_email/signals.py | 10 ++++- .../bulk_email/tests/test_signals.py | 40 +++++++++++++++++++ 2 files changed, 49 insertions(+), 1 deletion(-) diff --git a/lms/djangoapps/bulk_email/signals.py b/lms/djangoapps/bulk_email/signals.py index 7402dca75482..da18a459aeaa 100644 --- a/lms/djangoapps/bulk_email/signals.py +++ b/lms/djangoapps/bulk_email/signals.py @@ -7,6 +7,7 @@ from eventtracking import tracker from common.djangoapps.student.models import CourseEnrollment +from openedx.core.djangoapps.content.course_overviews.models import CourseOverview from openedx.core.djangoapps.user_api.accounts.signals import USER_RETIRE_MAILINGS from edx_ace.signals import ACE_MESSAGE_SENT @@ -27,7 +28,14 @@ def force_optout_all(sender, **kwargs): # lint-amnesty, pylint: disable=unused- raise TypeError('Expected a User type, but received None.') for enrollment in CourseEnrollment.objects.filter(user=user): - Optout.objects.get_or_create(user=user, course_id=enrollment.course.id) + try: + Optout.objects.get_or_create(user=user, course_id=enrollment.course.id) + except CourseOverview.DoesNotExist: + log.warning( + f"CourseOverview not found for enrollment {enrollment.id} (user: {user.id}), " + f"skipping optout creation. This may mean the course was deleted." + ) + continue @receiver(ACE_MESSAGE_SENT) diff --git a/lms/djangoapps/bulk_email/tests/test_signals.py b/lms/djangoapps/bulk_email/tests/test_signals.py index 1a3715284b12..01ad9312da4c 100644 --- a/lms/djangoapps/bulk_email/tests/test_signals.py +++ b/lms/djangoapps/bulk_email/tests/test_signals.py @@ -10,9 +10,11 @@ from django.core.management import call_command from django.urls import reverse +from common.djangoapps.student.models import CourseEnrollment from common.djangoapps.student.tests.factories import AdminFactory, CourseEnrollmentFactory, UserFactory from lms.djangoapps.bulk_email.models import BulkEmailFlag, Optout from lms.djangoapps.bulk_email.signals import force_optout_all +from opaque_keys.edx.keys import CourseKey from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase # lint-amnesty, pylint: disable=wrong-import-order from xmodule.modulestore.tests.factories import CourseFactory # lint-amnesty, pylint: disable=wrong-import-order @@ -85,3 +87,41 @@ def test_optout_course(self): assert len(mail.outbox) == 1 assert len(mail.outbox[0].to) == 1 assert mail.outbox[0].to[0] == self.instructor.email + + @patch('lms.djangoapps.bulk_email.signals.log.warning') + def test_optout_handles_missing_course_overview(self, mock_log_warning): + """ + Test that force_optout_all gracefully handles CourseEnrollments + with missing CourseOverview records + """ + # Create a course key for a course that doesn't exist in CourseOverview + nonexistent_course_key = CourseKey.from_string('course-v1:TestX+Missing+2023') + + # Create an enrollment with a course_id that doesn't have a CourseOverview + CourseEnrollment.objects.create( + user=self.student, + course_id=nonexistent_course_key, + mode='honor' + ) + + # Verify the orphaned enrollment exists + assert CourseEnrollment.objects.filter( + user=self.student, + course_id=nonexistent_course_key + ).exists() + + force_optout_all(sender=self.__class__, user=self.student) + + # Verify that a warning was logged for the missing CourseOverview + mock_log_warning.assert_called() + call_args = mock_log_warning.call_args[0][0] + assert "CourseOverview not found for enrollment" in call_args + assert f"user: {self.student.id}" in call_args + assert "skipping optout creation" in call_args + + # Verify that optouts were created for valid courses only + valid_course_optouts = Optout.objects.filter(user=self.student, course_id=self.course.id) + missing_course_optouts = Optout.objects.filter(user=self.student, course_id=nonexistent_course_key) + + assert valid_course_optouts.count() == 1 + assert missing_course_optouts.count() == 0 From 1917f35a00f54697cff6bd722756181dafa17375 Mon Sep 17 00:00:00 2001 From: Vivek Date: Tue, 13 Jan 2026 22:32:56 +0530 Subject: [PATCH 171/351] fix: restrict PDF rendering to relative paths (#64) * fix: restrict PDF rendering to relative paths * fix: enhance PDF viewer security by removing file param and communicating via postMessage * fix: remove HTML escaping from chapter URLs and titles in PDF viewer * fix: refactor PDF viewer integration for improved chapter navigation and security * fix: enhance security in StaticPdfBookTest by removing file parameter exposure * fix: secure PDF viewer iframe by removing unnecessary file parameter * fix: add newline at end of static_pdfbook.html for proper file termination --------- Co-authored-by: papphelix --- lms/djangoapps/staticbook/tests.py | 48 +++++--- lms/djangoapps/staticbook/views.py | 9 +- lms/templates/pdf_viewer.html | 180 +++++++++++++++++------------ lms/templates/static_pdfbook.html | 79 ++++++++----- 4 files changed, 193 insertions(+), 123 deletions(-) diff --git a/lms/djangoapps/staticbook/tests.py b/lms/djangoapps/staticbook/tests.py index a4e0d09bf096..1d42ac4b5bcc 100644 --- a/lms/djangoapps/staticbook/tests.py +++ b/lms/djangoapps/staticbook/tests.py @@ -129,8 +129,11 @@ def test_book(self): url = self.make_url('pdf_book', book_index=0) response = self.client.get(url) self.assertContains(response, "Chapter 1 for PDF") - self.assertNotContains(response, "options.chapterNum =") - self.assertNotContains(response, "page=") + # Verify file parameter is not present (security fix) + self.assertNotContains(response, "file=") + # Verify postMessage infrastructure is in place + self.assertContains(response, "request_pdf_url") + self.assertContains(response, "pdf_url_response") def test_book_chapter(self): # We can access a book at a particular chapter. @@ -138,8 +141,10 @@ def test_book_chapter(self): url = self.make_url('pdf_book', book_index=0, chapter=2) response = self.client.get(url) self.assertContains(response, "Chapter 2 for PDF") - self.assertNotContains(response, "file={}".format(PDF_BOOK['chapters'][1]['url'])) - self.assertNotContains(response, "page=") + # Verify file parameter is not present anywhere (security fix) + self.assertNotContains(response, "file=") + # Verify postMessage infrastructure is in place + self.assertContains(response, "request_pdf_url") def test_book_page(self): # We can access a book at a particular page. @@ -147,8 +152,10 @@ def test_book_page(self): url = self.make_url('pdf_book', book_index=0, page=17) response = self.client.get(url) self.assertContains(response, "Chapter 1 for PDF") - self.assertNotContains(response, "options.chapterNum =") - self.assertNotContains(response, "page=17") + # Verify file parameter is not present (security fix) + self.assertNotContains(response, "file=") + # Page parameter is still used in viewer_params + self.assertContains(response, "page=17") def test_book_chapter_page(self): # We can access a book at a particular chapter and page. @@ -156,8 +163,10 @@ def test_book_chapter_page(self): url = self.make_url('pdf_book', book_index=0, chapter=2, page=17) response = self.client.get(url) self.assertContains(response, "Chapter 2 for PDF") - self.assertNotContains(response, "file={}".format(PDF_BOOK['chapters'][1]['url'])) - self.assertNotContains(response, "page=17") + # Verify file parameter is not present (security fix) + self.assertNotContains(response, "file=") + # Page parameter is still used in viewer_params + self.assertContains(response, "page=17") def test_bad_book_id(self): # If the book id isn't an int, we'll get a 404. @@ -202,29 +211,32 @@ def test_chapter_page_xss(self): def test_static_url_map_contentstore(self): """ - This ensure static URL mapping is happening properly for - a course that uses the contentstore + This ensure static URL mapping is happening properly for + a course that uses the contentstore. + URLs are remapped in backend but not exposed via file parameter (security fix). """ self.make_course(pdf_textbooks=[PORTABLE_PDF_BOOK]) url = self.make_url('pdf_book', book_index=0, chapter=1) response = self.client.get(url) - self.assertNotContains(response, 'file={}'.format(PORTABLE_PDF_BOOK['chapters'][0]['url'])) - self.assertContains(response, 'file=/asset-v1:{0.org}+{0.course}+{0.run}+type@asset+block/{1}'.format( + # Verify file parameter is not present in response (security fix) + self.assertNotContains(response, 'file=') + # Verify the chapter URL is in the sidebar for postMessage communication + self.assertContains(response, '/asset-v1:{0.org}+{0.course}+{0.run}+type@asset+block/{1}'.format( self.course.location, PORTABLE_PDF_BOOK['chapters'][0]['url'].replace('/static/', ''))) def test_static_url_map_static_asset_path(self): """ - Like above, but used when the course has set a static_asset_path + Like above, but used when the course has set a static_asset_path. + URLs are remapped in backend but not exposed via file parameter (security fix). """ self.make_course(pdf_textbooks=[PORTABLE_PDF_BOOK], static_asset_path='awesomesauce') url = self.make_url('pdf_book', book_index=0, chapter=1) response = self.client.get(url) - self.assertNotContains(response, 'file={}'.format(PORTABLE_PDF_BOOK['chapters'][0]['url'])) - self.assertNotContains(response, 'file=/c4x/{0.org}/{0.course}/asset/{1}'.format( - self.course.location, - PORTABLE_PDF_BOOK['chapters'][0]['url'].replace('/static/', ''))) - self.assertContains(response, 'file=/static/awesomesauce/{}'.format( + # Verify file parameter is not present anywhere (security fix) + self.assertNotContains(response, 'file=') + # Verify the remapped URL is in the sidebar for postMessage communication + self.assertContains(response, '/static/awesomesauce/{}'.format( PORTABLE_PDF_BOOK['chapters'][0]['url'].replace('/static/', ''))) def test_invalid_chapter_id(self): diff --git a/lms/djangoapps/staticbook/views.py b/lms/djangoapps/staticbook/views.py index a9a6e922fa08..5c32f35f3b48 100644 --- a/lms/djangoapps/staticbook/views.py +++ b/lms/djangoapps/staticbook/views.py @@ -92,24 +92,21 @@ def pdf_index(request, course_id, book_index, chapter=None, page=None): if 'url' in textbook: textbook['url'] = remap_static_url(textbook['url'], course) current_url = textbook['url'] - if not current_url.startswith(('http://', 'https://')): - viewer_params = '&file=' - viewer_params += current_url # then remap all the chapter URLs as well, if they are provided. current_chapter = None if 'chapters' in textbook: for entry in textbook['chapters']: entry['url'] = remap_static_url(entry['url'], course) + # Security: Validate chapter URL doesn't contain dangerous schemes + if entry['url'].lower().startswith(('javascript:', 'data:', 'vbscript:', 'file:')): + entry['url'] = '' # Sanitize dangerous URLs if chapter is not None and int(chapter) <= (len(textbook['chapters'])): current_chapter = textbook['chapters'][int(chapter) - 1] else: current_chapter = textbook['chapters'][0] current_url = current_chapter['url'] - if not current_url.startswith(('http://', 'https://')): - viewer_params = '&file=' - viewer_params += current_url viewer_params += '#zoom=page-fit&disableRange=true' if page is not None: diff --git a/lms/templates/pdf_viewer.html b/lms/templates/pdf_viewer.html index a3314d54b5ac..1f4b8cd54abf 100644 --- a/lms/templates/pdf_viewer.html +++ b/lms/templates/pdf_viewer.html @@ -1,5 +1,5 @@ <%page expression_filter="h"/> - + <%namespace name='static' file='static_content.html'/> <%! from openedx.core.djangolib.js_utils import ( @@ -46,10 +46,44 @@ PDFJS.workerSrc = "${static.url('js/vendor/pdfjs/pdf.worker.js') | n, js_escaped_string}"; PDFJS.disableWorker = true; PDFJS.cMapUrl = "${static.url('css/vendor/pdfjs/cmaps/') | n, js_escaped_string}"; - PDF_URL = '${current_url | n, js_escaped_string}'; + + var PDF_URL = '${current_url | n, js_escaped_string}'; + + if (window.parent !== window) { + window.parent.postMessage({type: 'request_pdf_url'}, '*'); + + function handlePdfUrlResponse(event) { + if (event.data && event.data.type === 'pdf_url_response') { + PDF_URL = event.data.url; + + if (PDFViewerApplication.open) { + PDFViewerApplication.open(PDF_URL); + PDFViewerApplication.mouseScroll(0); + + setTimeout(function() { + if (PDFViewerApplication.pdfDocument) { + if (event.data.title) document.getElementById('titleField').textContent = event.data.title; + if (event.data.author) document.getElementById('authorField').textContent = event.data.author; + if (event.data.subject) document.getElementById('subjectField').textContent = event.data.subject; + if (event.data.keywords) document.getElementById('keywordsField').textContent = event.data.keywords; + document.getElementById('creatorField').textContent = 'edX Platform'; + } + }, 500); + } + } else if (event.data && event.data.type === 'chapter_change') { + window.parent.postMessage({type: 'request_pdf_url'}, '*'); + } + } + + window.addEventListener('message', handlePdfUrlResponse); + } + + document.addEventListener('DOMContentLoaded', function () { + PDFViewerApplication && PDFViewerApplication.open(PDF_URL); + }); - + <%static:js group='main_vendor'/> <%static:js group='application'/> @@ -347,77 +381,77 @@

    -