𝓢𝓽𝓾𝓭𝓲𝓸 Home
', - status_code=200, - html=True) - - @override_waffle_flag(toggles.LEGACY_STUDIO_HOME, True) - def test_course_explicit_english(self): - """Test viewing the index page with no courses""" - self.client = AjaxEnabledTestClient() # lint-amnesty, pylint: disable=attribute-defined-outside-init - self.client.login(username=self.uname, password=self.password) - - resp = self.client.get_html( - '/home/', - {}, - HTTP_ACCEPT_LANGUAGE='en', - ) - - self.assertContains(resp, - '𝓢𝓽𝓾𝓭𝓲𝓸 Home
', - status_code=200, - html=True) - - # **** - # NOTE: - # **** - # - # This test will break when we replace this fake 'test' language - # with actual Esperanto. This test will need to be updated with - # actual Esperanto at that time. - # Test temporarily disable since it depends on creation of dummy strings - @skip - def test_course_with_accents(self): - """Test viewing the index page with no courses""" - self.client = AjaxEnabledTestClient() # lint-amnesty, pylint: disable=attribute-defined-outside-init - self.client.login(username=self.uname, password=self.password) - - resp = self.client.get_html( - '/home/', - {}, - HTTP_ACCEPT_LANGUAGE='eo' - ) - - TEST_STRING = ( - '' - 'My \xc7\xf6\xfcrs\xe9s L#' - '
' - ) - - self.assertContains(resp, - TEST_STRING, - status_code=200, - html=True) diff --git a/cms/djangoapps/contentstore/tests/test_upstream_downstream_links.py b/cms/djangoapps/contentstore/tests/test_upstream_downstream_links.py index 90fec8471651..5c3ba8386480 100644 --- a/cms/djangoapps/contentstore/tests/test_upstream_downstream_links.py +++ b/cms/djangoapps/contentstore/tests/test_upstream_downstream_links.py @@ -14,7 +14,7 @@ from common.djangoapps.student.tests.factories import UserFactory from openedx.core.djangolib.testing.utils import skip_unless_cms -from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase +from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase, ImmediateOnCommitMixin from xmodule.modulestore.tests.factories import BlockFactory, CourseFactory from ..models import ContainerLink, LearningContextLinksStatus, LearningContextLinksStatusChoices, ComponentLink @@ -265,7 +265,12 @@ def test_call_for_nonexistent_course(self): @skip_unless_cms -class TestUpstreamLinksEvents(ModuleStoreTestCase, OpenEdxEventsTestMixin, BaseUpstreamLinksHelpers): +class TestUpstreamLinksEvents( + ImmediateOnCommitMixin, + ModuleStoreTestCase, + OpenEdxEventsTestMixin, + BaseUpstreamLinksHelpers, +): """ Test signals related to managing upstream->downstream links. """ diff --git a/cms/djangoapps/contentstore/tests/test_video_audio_description_handler.py b/cms/djangoapps/contentstore/tests/test_video_audio_description_handler.py new file mode 100644 index 000000000000..617bc90d5132 --- /dev/null +++ b/cms/djangoapps/contentstore/tests/test_video_audio_description_handler.py @@ -0,0 +1,199 @@ +""" +Tests for the studio_audio_description XBlock handler and its +contentstore.enable_audio_description waffle flag gate. + +The handler itself delegates to the storage helpers in +cms.djangoapps.contentstore.audio_description_storage_handlers; these +tests focus on the handler's gating behavior and dispatch logic, not on +the storage helpers themselves. + +These tests live in CMS-test land (rather than alongside the rest of +the LMS-side video handler tests in +lms/djangoapps/courseware/tests/test_video_handlers.py) because +cms.djangoapps.contentstore.toggles transitively imports the +Studio-only search-api and so cannot be loaded under LMS test settings. +""" + +import importlib +from unittest.mock import Mock, patch + +from django.test import TestCase +from edx_toggles.toggles.testutils import override_waffle_flag +from opaque_keys.edx.locator import CourseLocator +from webob import Request + +from cms.djangoapps.contentstore.toggles import ENABLE_AUDIO_DESCRIPTION +from xmodule.video_block.video_block import VideoBlock + + +class StudioAudioDescriptionHandlerTest(TestCase): + """ + The XBlock @handler decorator does not wrap the function -- it just + sets _is_xblock_handler = True -- so we can call + VideoBlock.studio_audio_description as a plain function with a Mock + standing in for `self`. The handler only touches self.edx_video_id + and self.audio_description, both of which the Mock can carry. + """ + + def setUp(self): + super().setUp() + + importlib.import_module("cms.djangoapps.contentstore.views") + self.storage_handlers = importlib.import_module( + "cms.djangoapps.contentstore.audio_description_storage_handlers" + ) + + def _build_block_mock(self, edx_video_id="video-1", audio_description=""): + """ + Return a minimal Mock that satisfies the attribute contract + expected by the studio_audio_description handler. + """ + block = Mock( + spec_set=[ + "edx_video_id", + "audio_description", + "audio_description_video_id", + "course_id", + ] + ) + block.edx_video_id = edx_video_id + block.audio_description = audio_description + block.audio_description_video_id = "" + block.course_id = CourseLocator(org="test", course="test", run="test") + return block + + def _call(self, block, method, body=None, request=None): + """ + Build a WebOb Request and invoke the handler directly, + bypassing the XBlock runtime dispatch machinery. + """ + if request is not None: + return VideoBlock.studio_audio_description(block, request=request) + kwargs = {"method": method} + if body is not None: + kwargs["body"] = body + request = Request.blank("", **kwargs) + return VideoBlock.studio_audio_description(block, request=request) + + @override_waffle_flag(ENABLE_AUDIO_DESCRIPTION, active=False) + def test_handler_returns_404_when_flag_disabled(self): + """ + When the upload flag is off, every HTTP method on the handler + must return 404 so the endpoint looks non-existent to clients. + """ + block = self._build_block_mock() + for method in ("GET", "POST", "DELETE"): + response = self._call(block, method) + self.assertEqual(response.status_code, 404, msg=f"method={method}") + + @override_waffle_flag(ENABLE_AUDIO_DESCRIPTION, active=True) + def test_post_uploads_file_and_returns_url(self): + """ + With the flag on, a POST request carrying a file should reach + upload_audio_description and return {file_name, url} with 201. + """ + block = self._build_block_mock(edx_video_id="video-1") + + file_mock = Mock() + file_mock.name = "bar.mp3" + file_mock.type = "audio/mpeg" + file_mock.file = Mock() + + request = Mock() + request.method = "POST" + request.POST = { + "file": file_mock, + "file_name": "bar.mp3", + "content_type": "audio/mpeg", + } + + with patch.object( + self.storage_handlers, "upload_audio_description" + ) as mock_upload: + mock_upload.return_value = "https://s3.example/bar.mp3" + response = self._call(block, "POST", request=request) + + self.assertEqual(response.status_code, 201) + self.assertEqual( + response.json, + {"file_name": "bar.mp3", "url": "https://s3.example/bar.mp3"}, + ) + mock_upload.assert_called_once_with( + edx_video_id="video-1", + file_name="bar.mp3", + content_type="audio/mpeg", + file_data=file_mock.file, + ) + + @override_waffle_flag(ENABLE_AUDIO_DESCRIPTION, active=True) + def test_post_returns_400_when_file_missing(self): + """ + With the flag on but no file in the POST body, the handler must + return 400 with an error message. + """ + block = self._build_block_mock(edx_video_id="video-1") + + request = Mock() + request.method = "POST" + request.POST = {} # no 'file' key + + response = self._call(block, "POST", request=request) + + self.assertEqual(response.status_code, 400) + self.assertIn("error", response.json) + + @override_waffle_flag(ENABLE_AUDIO_DESCRIPTION, active=True) + def test_get_returns_404_when_no_url(self): + """ + With the flag on but no AD record on the block, the GET branch + should return 404 (the storage helper returns None). + """ + block = self._build_block_mock() + + with patch.object(self.storage_handlers, "get_audio_description_url") as mock_url: + mock_url.return_value = None + response = self._call(block, "GET") + + self.assertEqual(response.status_code, 404) + + @override_waffle_flag(ENABLE_AUDIO_DESCRIPTION, active=True) + def test_get_returns_url_when_present(self): + """ + With the flag on and a ready AD record, the GET branch returns + a JSON body containing the helper's pre-signed URL plus the + block's stored filename. + """ + block = self._build_block_mock(audio_description="bar.mp3") + + with patch.object(self.storage_handlers, "get_audio_description_url") as mock_url: + mock_url.return_value = "https://s3.example/get-presigned" + response = self._call(block, "GET") + + self.assertEqual(response.status_code, 200) + self.assertEqual( + response.json, + { + "file_name": "bar.mp3", + "url": "https://s3.example/get-presigned", + }, + ) + + @override_waffle_flag(ENABLE_AUDIO_DESCRIPTION, active=True) + def test_delete_when_flag_enabled(self): + """ + With the flag on, a DELETE request should call the storage + helper, clear the block's audio_description field, and return + 204. + """ + block = self._build_block_mock( + edx_video_id="video-1", audio_description="bar.mp3" + ) + + with patch.object( + self.storage_handlers, "delete_audio_description" + ) as mock_delete: + response = self._call(block, "DELETE") + + self.assertEqual(response.status_code, 204) + self.assertEqual(block.audio_description, "") + mock_delete.assert_called_once_with("video-1") diff --git a/cms/djangoapps/contentstore/tests/tests.py b/cms/djangoapps/contentstore/tests/tests.py index 40b0f8ad4cbf..d151b1d58526 100644 --- a/cms/djangoapps/contentstore/tests/tests.py +++ b/cms/djangoapps/contentstore/tests/tests.py @@ -11,7 +11,7 @@ import datetime import time from unittest import mock -from urllib.parse import quote_plus +from urllib.parse import quote_plus, unquote from ddt import data, ddt, unpack from django.conf import settings @@ -24,6 +24,7 @@ from cms.djangoapps.contentstore import toggles from cms.djangoapps.contentstore.tests.test_course_settings import CourseTestCase from cms.djangoapps.contentstore.tests.utils import AjaxEnabledTestClient, parse_json, registration, user +from cms.djangoapps.contentstore.utils import get_studio_home_url from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase # lint-amnesty, pylint: disable=wrong-import-order from xmodule.modulestore.tests.factories import CourseFactory # lint-amnesty, pylint: disable=wrong-import-order @@ -114,12 +115,6 @@ def setUp(self): # clear the cache so ratelimiting won't affect these tests cache.clear() - def check_page_get(self, url, expected): - resp = self.client.get_html(url) - self.assertEqual(resp.status_code, expected) - return resp - - @override_waffle_flag(toggles.LEGACY_STUDIO_HOME, True) def test_private_pages_auth(self): """Make sure pages that do require login work.""" auth_pages = ( @@ -143,7 +138,9 @@ def test_private_pages_auth(self): print('Not logged in') for page in auth_pages: print(f"Checking '{page}'") - self.check_page_get(page, expected=302) + resp = self.client.get_html(page) + assert resp.status_code == 302 + assert resp.url == unquote(reverse("login", query={"next": page})) # Logged in should work. self.login(self.email, self.pw) @@ -151,10 +148,11 @@ def test_private_pages_auth(self): print('Logged in') for page in simple_auth_pages: print(f"Checking '{page}'") - self.check_page_get(page, expected=200) + resp = self.client.get_html(page) + assert resp.status_code == 302 + assert resp.url == get_studio_home_url() @override_settings(SESSION_INACTIVITY_TIMEOUT_IN_SECONDS=1) - @override_waffle_flag(toggles.LEGACY_STUDIO_HOME, True) def test_inactive_session_timeout(self): """ Verify that an inactive session times out and redirects to the @@ -168,7 +166,8 @@ def test_inactive_session_timeout(self): # make sure we can access courseware immediately course_url = '/home/' resp = self.client.get_html(course_url) - self.assertEqual(resp.status_code, 200) + assert resp.status_code == 302 + assert resp.url == get_studio_home_url() # then wait a bit and see if we get timed out time.sleep(2) diff --git a/cms/djangoapps/contentstore/toggles.py b/cms/djangoapps/contentstore/toggles.py index c287f8c4dbec..03c374fd1b94 100644 --- a/cms/djangoapps/contentstore/toggles.py +++ b/cms/djangoapps/contentstore/toggles.py @@ -2,7 +2,6 @@ CMS feature toggles. """ from edx_toggles.toggles import SettingDictToggle, WaffleFlag -from openedx.core.djangoapps.content.search import api as search_api from openedx.core.djangoapps.waffle_utils import CourseWaffleFlag @@ -67,42 +66,47 @@ def bypass_olx_failure_enabled(): return BYPASS_OLX_FAILURE.is_enabled() -# .. toggle_name: legacy_studio.exam_settings -# .. toggle_implementation: WaffleFlag +# .. toggle_name: contentstore.enable_audio_description +# .. toggle_implementation: CourseWaffleFlag # .. toggle_default: False -# .. toggle_description: Temporarily fall back to the old proctored exam settings view. -# .. toggle_use_cases: temporary -# .. toggle_creation_date: 2025-03-14 -# .. toggle_target_removal_date: 2025-09-14 -# .. toggle_tickets: https://github.com/openedx/edx-platform/issues/36275 -# .. toggle_warning: In Ulmo, this toggle will be removed. Only the new (React-based) experience will be available. -LEGACY_STUDIO_EXAM_SETTINGS = CourseWaffleFlag("legacy_studio.exam_settings", __name__) +# .. toggle_description: Enables the audio description (AD) upload UI in the +# Studio video editor and the corresponding studio_audio_description XBlock +# handler. When disabled, course authors cannot upload, replace, or delete +# audio description files for video blocks. Playback of AD files that were +# already uploaded is NOT gated by this flag and continues to work in the +# LMS -- disable the playback path with a separate flag if needed. +# .. toggle_use_cases: open_edx +# .. toggle_creation_date: 2026-04-07 +ENABLE_AUDIO_DESCRIPTION = CourseWaffleFlag( + f'{CONTENTSTORE_NAMESPACE}.enable_audio_description', + __name__, +) -def exam_setting_view_enabled(course_key): +def audio_description_enabled(course_key): """ - Returns a boolean if proctoring exam setting mfe view is enabled. + Return True if the audio description upload UI and handler are enabled. """ - return not LEGACY_STUDIO_EXAM_SETTINGS.is_enabled(course_key) + return ENABLE_AUDIO_DESCRIPTION.is_enabled(course_key) -# .. toggle_name: legacy_studio.text_editor +# .. toggle_name: legacy_studio.exam_settings # .. toggle_implementation: WaffleFlag # .. toggle_default: False -# .. toggle_description: Temporarily fall back to the old Text component (a.k.a. html block) editor. +# .. toggle_description: Temporarily fall back to the old proctored exam settings view. # .. toggle_use_cases: temporary # .. toggle_creation_date: 2025-03-14 # .. toggle_target_removal_date: 2025-09-14 # .. toggle_tickets: https://github.com/openedx/edx-platform/issues/36275 # .. toggle_warning: In Ulmo, this toggle will be removed. Only the new (React-based) experience will be available. -LEGACY_STUDIO_TEXT_EDITOR = CourseWaffleFlag("legacy_studio.text_editor", __name__) +LEGACY_STUDIO_EXAM_SETTINGS = CourseWaffleFlag("legacy_studio.exam_settings", __name__) -def use_new_text_editor(course_key): +def exam_setting_view_enabled(course_key): """ - Returns a boolean = true if new text editor is enabled + Returns a boolean if proctoring exam setting mfe view is enabled. """ - return not LEGACY_STUDIO_TEXT_EDITOR.is_enabled(course_key) + return not LEGACY_STUDIO_EXAM_SETTINGS.is_enabled(course_key) # .. toggle_name: legacy_studio.video_editor @@ -181,25 +185,6 @@ def individualize_anonymous_user_id(course_id): return INDIVIDUALIZE_ANONYMOUS_USER_ID.is_enabled(course_id) -# .. toggle_name: legacy_studio.home -# .. toggle_implementation: WaffleFlag -# .. toggle_default: False -# .. toggle_description: Temporarily fall back to the old Studio logged-in landing page. -# .. toggle_use_cases: temporary -# .. toggle_creation_date: 2025-03-14 -# .. toggle_target_removal_date: 2025-09-14 -# .. toggle_tickets: https://github.com/openedx/edx-platform/issues/36275 -# .. toggle_warning: In Ulmo, this toggle will be removed. Only the new (React-based) experience will be available. -LEGACY_STUDIO_HOME = WaffleFlag('legacy_studio.home', __name__) - - -def use_new_home_page(): - """ - Returns a boolean if new studio home page mfe is enabled - """ - return not LEGACY_STUDIO_HOME.is_enabled() - - # .. toggle_name: legacy_studio.custom_pages # .. toggle_implementation: WaffleFlag # .. toggle_default: False @@ -351,25 +336,6 @@ def use_new_export_page(course_key): return not LEGACY_STUDIO_EXPORT.is_enabled(course_key) -# .. toggle_name: legacy_studio.files_uploads -# .. toggle_implementation: WaffleFlag -# .. toggle_default: False -# .. toggle_description: Temporarily fall back to the old Studio Files & Uploads page. -# .. toggle_use_cases: temporary -# .. toggle_creation_date: 2025-03-14 -# .. toggle_target_removal_date: 2025-09-14 -# .. toggle_tickets: https://github.com/openedx/edx-platform/issues/36275 -# .. toggle_warning: In Ulmo, this toggle will be removed. Only the new (React-based) experience will be available. -LEGACY_STUDIO_FILES_UPLOADS = CourseWaffleFlag('legacy_studio.files_uploads', __name__) - - -def use_new_files_uploads_page(course_key): - """ - Returns a boolean if new studio files and uploads mfe is enabled - """ - return not LEGACY_STUDIO_FILES_UPLOADS.is_enabled(course_key) - - # .. toggle_name: contentstore.new_studio_mfe.use_new_video_uploads_page # .. toggle_implementation: CourseWaffleFlag # .. toggle_default: False @@ -402,13 +368,6 @@ def use_new_video_uploads_page(course_key): LEGACY_STUDIO_COURSE_OUTLINE = CourseWaffleFlag('legacy_studio.course_outline', __name__) -def use_new_course_outline_page(course_key): - """ - Returns a boolean if new studio course outline mfe is enabled - """ - return not LEGACY_STUDIO_COURSE_OUTLINE.is_enabled(course_key) - - # .. toggle_name: legacy_studio.unit_editor # .. toggle_implementation: WaffleFlag # .. toggle_default: False @@ -612,6 +571,7 @@ def libraries_v2_enabled(): Requires the ENABLE_CONTENT_LIBRARIES feature flag to be enabled, plus Meilisearch. """ + from openedx.core.djangoapps.content.search import api as search_api # pylint: disable=import-outside-toplevel return ( ENABLE_CONTENT_LIBRARIES.is_enabled() and search_api.is_meilisearch_enabled() and @@ -682,3 +642,45 @@ def enable_course_optimizer_check_prev_run_links(course_key): Returns a boolean if previous run course optimizer feature is enabled for the given course. """ return ENABLE_COURSE_OPTIMIZER_CHECK_PREV_RUN_LINKS.is_enabled(course_key) + + +# .. toggle_name: contentstore.enable_unit_expanded_view +# .. toggle_implementation: CourseWaffleFlag +# .. toggle_default: False +# .. toggle_description: When enabled, the Unit Expanded View feature in the Course Outline is activated. +# .. toggle_use_cases: temporary +# .. toggle_creation_date: 2026-01-01 +# .. toggle_target_removal_date: 2026-06-01 +# .. toggle_tickets: TNL2-473 +ENABLE_UNIT_EXPANDED_VIEW = CourseWaffleFlag( + f"{CONTENTSTORE_NAMESPACE}.enable_unit_expanded_view", __name__ +) + + +def enable_unit_expanded_view(course_key): + """ + Returns a boolean if the Unit Expanded View feature is enabled for the given course. + """ + return ENABLE_UNIT_EXPANDED_VIEW.is_enabled(course_key) + + +# .. toggle_name: contentstore.enable_outline_component_creation +# .. toggle_implementation: CourseWaffleFlag +# .. toggle_default: False +# .. toggle_description: When enabled, the Add Component widget is displayed inside each +# .. expanded unit on the Course Outline page, allowing authors to add new XBlocks +# .. directly without navigating to the unit page. +# .. toggle_use_cases: temporary +# .. toggle_creation_date: 2026-03-24 +# .. toggle_target_removal_date: 2026-09-24 +# .. toggle_tickets: TNL2-533 +ENABLE_OUTLINE_COMPONENT_CREATION = CourseWaffleFlag( + f"{CONTENTSTORE_NAMESPACE}.enable_outline_component_creation", __name__ +) + + +def enable_outline_component_creation(course_key): + """ + Returns a boolean if the Add Component in Outline feature is enabled for the given course. + """ + return ENABLE_OUTLINE_COMPONENT_CREATION.is_enabled(course_key) diff --git a/cms/djangoapps/contentstore/utils.py b/cms/djangoapps/contentstore/utils.py index 3ca1d20bf564..5506d8c33e41 100644 --- a/cms/djangoapps/contentstore/utils.py +++ b/cms/djangoapps/contentstore/utils.py @@ -15,7 +15,7 @@ from bs4 import BeautifulSoup from django.conf import settings -from django.core.exceptions import ObjectDoesNotExist, ValidationError +from django.core.exceptions import ImproperlyConfigured, ObjectDoesNotExist, ValidationError from django.urls import reverse from django.utils import translation from django.utils.text import Truncator @@ -43,26 +43,22 @@ split_library_view_on_dashboard, use_new_advanced_settings_page, use_new_certificates_page, - use_new_course_outline_page, use_new_course_team_page, use_new_custom_pages, use_new_export_page, - use_new_files_uploads_page, use_new_grading_page, use_new_group_configurations_page, - use_new_home_page, use_new_import_page, use_new_schedule_details_page, - use_new_text_editor, use_new_textbooks_page, use_new_unit_page, use_new_updates_page, - use_new_video_editor, use_new_video_uploads_page, ) from cms.djangoapps.models.settings.course_grading import CourseGradingModel from cms.djangoapps.models.settings.course_metadata import CourseMetadata -from cms.djangoapps.modulestore_migrator.api import get_migration_info +from cms.djangoapps.modulestore_migrator import api as migrator_api +from cms.djangoapps.modulestore_migrator.data import ModulestoreMigration from common.djangoapps.course_action_state.managers import CourseActionStateItemNotFoundError from common.djangoapps.course_action_state.models import CourseRerunState, CourseRerunUIStateManager from common.djangoapps.course_modes.models import CourseMode @@ -117,6 +113,7 @@ get_all_partitions_for_course, # lint-amnesty, pylint: disable=wrong-import-order ) from xmodule.services import ConfigurationService, SettingsService, TeamsConfigurationService +from xmodule.util.keys import BlockKey from .models import ComponentLink, ContainerLink @@ -288,11 +285,10 @@ def get_editor_page_base_url(course_locator) -> str: Gets course authoring microfrontend URL for links to the new base editors """ editor_url = None - if use_new_text_editor(course_locator) or use_new_video_editor(course_locator): - mfe_base_url = get_course_authoring_url(course_locator) - course_mfe_url = f'{mfe_base_url}/course/{course_locator}/editor' - if mfe_base_url: - editor_url = course_mfe_url + mfe_base_url = get_course_authoring_url(course_locator) + course_mfe_url = f'{mfe_base_url}/course/{course_locator}/editor' + if mfe_base_url: + editor_url = course_mfe_url return editor_url @@ -300,12 +296,15 @@ def get_studio_home_url(): """ Gets course authoring microfrontend URL for Studio Home view. """ - studio_home_url = None - if use_new_home_page(): - mfe_base_url = settings.COURSE_AUTHORING_MICROFRONTEND_URL - if mfe_base_url: - studio_home_url = f'{mfe_base_url}/home' - return studio_home_url + mfe_base_url = settings.COURSE_AUTHORING_MICROFRONTEND_URL + if mfe_base_url: + studio_home_url = f'{mfe_base_url}/home' + return studio_home_url + + raise ImproperlyConfigured( + "The COURSE_AUTHORING_MICROFRONTEND_URL must be configured. " + "Please set it to the base url for your authoring MFE." + ) def get_schedule_details_url(course_locator) -> str: @@ -417,11 +416,10 @@ def get_files_uploads_url(course_locator) -> str: Gets course authoring microfrontend URL for files and uploads page view. """ files_uploads_url = None - if use_new_files_uploads_page(course_locator): - mfe_base_url = get_course_authoring_url(course_locator) - course_mfe_url = f'{mfe_base_url}/course/{course_locator}/assets' - if mfe_base_url: - files_uploads_url = course_mfe_url + mfe_base_url = get_course_authoring_url(course_locator) + course_mfe_url = f'{mfe_base_url}/course/{course_locator}/assets' + if mfe_base_url: + files_uploads_url = course_mfe_url return files_uploads_url @@ -443,13 +441,12 @@ def get_course_outline_url(course_locator, block_to_show=None) -> str: Gets course authoring microfrontend URL for course oultine page view. """ course_outline_url = None - if use_new_course_outline_page(course_locator): - mfe_base_url = get_course_authoring_url(course_locator) - course_mfe_url = f'{mfe_base_url}/course/{course_locator}' - if block_to_show: - course_mfe_url += f'?show={quote_plus(block_to_show)}' - if mfe_base_url: - course_outline_url = course_mfe_url + mfe_base_url = get_course_authoring_url(course_locator) + course_mfe_url = f'{mfe_base_url}/course/{course_locator}' + if block_to_show: + course_mfe_url += f'?show={quote_plus(block_to_show)}' + if mfe_base_url: + course_outline_url = course_mfe_url return course_outline_url @@ -1582,13 +1579,12 @@ def request_response_format_is_json(request, response_format): def get_library_context(request, request_is_json=False): """ - Utils is used to get context of course home library tab. - It is used for both DRF and django views. + Utils is used to get context of course home library tab. Returned in DRF view. """ from cms.djangoapps.contentstore.views.course import ( _accessible_libraries_iter, - _format_library_for_view, _get_course_creator_status, + format_library_for_view, get_allowed_organizations, get_allowed_organizations_for_libraries, user_can_create_organizations, @@ -1600,21 +1596,25 @@ def get_library_context(request, request_is_json=False): user_can_create_library, ) + is_migrated: bool | None # None means: do not filter on is_migrated + if (is_migrated_param := request.GET.get('is_migrated')) is not None: + is_migrated = BooleanField().to_internal_value(is_migrated_param) + else: + is_migrated = None libraries = list(_accessible_libraries_iter(request.user) if libraries_v1_enabled() else []) - library_keys = [lib.location.library_key for lib in libraries] - migration_info = get_migration_info(library_keys) - is_migrated_filter = request.GET.get('is_migrated', None) + migration_info: dict[LibraryLocator, ModulestoreMigration | None] = { + lib.id: migrator_api.get_forwarding(lib.id) + for lib in libraries + } data = { 'libraries': [ - _format_library_for_view( + format_library_for_view( lib, request, - migrated_to=migration_info.get(lib.location.library_key) + migration=migration_info[lib.id], ) for lib in libraries - if is_migrated_filter is None or ( - BooleanField().to_internal_value(is_migrated_filter) == (lib.location.library_key in migration_info) - ) + if is_migrated is None or is_migrated == bool(migration_info[lib.id]) ] } @@ -1723,8 +1723,7 @@ def format_in_process_course_view(uca): def get_home_context(request, no_course=False): """ - Utils is used to get context of course home. - It is used for both DRF and django views. + Utils is used to get context of course home. Returned by DRF view. """ from cms.djangoapps.contentstore.views.course import ( @@ -2411,10 +2410,11 @@ def _create_or_update_component_link(created: datetime | None, xblock): top_level_parent_usage_key = None if xblock.top_level_downstream_parent_key is not None: + block_key = BlockKey.from_string(xblock.top_level_downstream_parent_key) top_level_parent_usage_key = BlockUsageLocator( xblock.usage_key.course_key, - xblock.top_level_downstream_parent_key.get('type'), - xblock.top_level_downstream_parent_key.get('id'), + block_key.type, + block_key.id, ) ComponentLink.update_or_create( @@ -2426,7 +2426,7 @@ def _create_or_update_component_link(created: datetime | None, xblock): top_level_parent_usage_key=top_level_parent_usage_key, version_synced=xblock.upstream_version, version_declined=xblock.upstream_version_declined, - downstream_is_modified=len(getattr(xblock, "downstream_customized", [])) > 0, + downstream_customized=getattr(xblock, "downstream_customized", []), created=created, ) @@ -2444,10 +2444,11 @@ def _create_or_update_container_link(created: datetime | None, xblock): top_level_parent_usage_key = None if xblock.top_level_downstream_parent_key is not None: + block_key = BlockKey.from_string(xblock.top_level_downstream_parent_key) top_level_parent_usage_key = BlockUsageLocator( xblock.usage_key.course_key, - xblock.top_level_downstream_parent_key.get('type'), - xblock.top_level_downstream_parent_key.get('id'), + block_key.type, + block_key.id, ) ContainerLink.update_or_create( @@ -2459,7 +2460,7 @@ def _create_or_update_container_link(created: datetime | None, xblock): version_synced=xblock.upstream_version, top_level_parent_usage_key=top_level_parent_usage_key, version_declined=xblock.upstream_version_declined, - downstream_is_modified=len(getattr(xblock, "downstream_customized", [])) > 0, + downstream_customized=getattr(xblock, "downstream_customized", []), created=created, ) diff --git a/cms/djangoapps/contentstore/video_storage_handlers.py b/cms/djangoapps/contentstore/video_storage_handlers.py index 87086c9951ac..b80fbdd80ce7 100644 --- a/cms/djangoapps/contentstore/video_storage_handlers.py +++ b/cms/djangoapps/contentstore/video_storage_handlers.py @@ -13,12 +13,11 @@ import shutil import pathlib import zipfile - from contextlib import closing from datetime import datetime, timedelta from uuid import uuid4 -from boto.s3.connection import S3Connection -from boto import s3 + +import boto3 from django.conf import settings from django.contrib.staticfiles.storage import staticfiles_storage from django.http import FileResponse, HttpResponseNotFound, StreamingHttpResponse @@ -33,8 +32,8 @@ create_video, get_3rd_party_transcription_plans, get_available_transcript_languages, - get_video_transcript_url, get_transcript_preferences, + get_video_transcript_url, get_videos_for_course, remove_transcript_preferences, remove_video_for_course, @@ -55,10 +54,7 @@ from common.djangoapps.util.json_request import JsonResponse from openedx.core.djangoapps.video_config.models import VideoTranscriptEnabledFlag from openedx.core.djangoapps.video_config.toggles import PUBLIC_VIDEO_SHARE -from openedx.core.djangoapps.video_pipeline.config.waffle import ( - DEPRECATE_YOUTUBE, - ENABLE_DEVSTACK_VIDEO_UPLOADS, -) +from openedx.core.djangoapps.video_pipeline.config.waffle import DEPRECATE_YOUTUBE from openedx.core.djangoapps.waffle_utils import CourseWaffleFlag from xmodule.modulestore.django import modulestore # lint-amnesty, pylint: disable=wrong-import-order @@ -812,7 +808,8 @@ def videos_post(course, request): if error: return {'error': error}, 400 - bucket = storage_service_bucket() + s3_client = boto3.client('s3') + req_files = data['files'] resp_files = [] @@ -826,7 +823,6 @@ def videos_post(course, request): return {'error': error_msg}, 400 edx_video_id = str(uuid4()) - key = storage_service_key(bucket, file_name=edx_video_id) metadata_list = [ ('client_video_id', file_name), @@ -846,12 +842,15 @@ def videos_post(course, request): if transcript_preferences is not None: metadata_list.append(('transcript_preferences', json.dumps(transcript_preferences))) - for metadata_name, value in metadata_list: - key.set_metadata(metadata_name, value) - upload_url = key.generate_url( - KEY_EXPIRATION_IN_SECONDS, - 'PUT', - headers={'Content-Type': req_file['content_type']} + upload_url = s3_client.generate_presigned_url( + ClientMethod='put_object', + Params={ + 'Bucket': storage_service_bucket_name(), + 'Key': storage_service_key_name(edx_video_id), + 'ContentType': req_file['content_type'], + 'Metadata': dict(metadata_list), + }, + ExpiresIn=KEY_EXPIRATION_IN_SECONDS, ) # persist edx_video_id in VAL @@ -869,41 +868,21 @@ def videos_post(course, request): return {'files': resp_files}, 200 -def storage_service_bucket(): +def storage_service_bucket_name(): """ - Returns an S3 bucket for video upload. + Returns name of S3 bucket to use for video upload. """ - if ENABLE_DEVSTACK_VIDEO_UPLOADS.is_enabled(): - params = { - 'aws_access_key_id': settings.AWS_ACCESS_KEY_ID, - 'aws_secret_access_key': settings.AWS_SECRET_ACCESS_KEY, - 'security_token': settings.AWS_SECURITY_TOKEN - - } - else: - params = { - 'aws_access_key_id': settings.AWS_ACCESS_KEY_ID, - 'aws_secret_access_key': settings.AWS_SECRET_ACCESS_KEY - } - - conn = S3Connection(**params) - - # We don't need to validate our bucket, it requires a very permissive IAM permission - # set since behind the scenes it fires a HEAD request that is equivalent to get_all_keys() - # meaning it would need ListObjects on the whole bucket, not just the path used in each - # environment (since we share a single bucket for multiple deployments in some configurations) - return conn.get_bucket(settings.VIDEO_UPLOAD_PIPELINE['VEM_S3_BUCKET'], validate=False) + return settings.VIDEO_UPLOAD_PIPELINE['VEM_S3_BUCKET'] -def storage_service_key(bucket, file_name): +def storage_service_key_name(file_name): """ - Returns an S3 key to the given file in the given bucket. + Returns the S3 object key to be used for a given video filename. """ - key_name = "{}/{}".format( + return "{}/{}".format( settings.VIDEO_UPLOAD_PIPELINE.get("ROOT_PATH", ""), file_name ) - return s3.key.Key(bucket, key_name) def send_video_status_update(updates): diff --git a/cms/djangoapps/contentstore/views/block.py b/cms/djangoapps/contentstore/views/block.py index b57042085df2..238627bc6618 100644 --- a/cms/djangoapps/contentstore/views/block.py +++ b/cms/djangoapps/contentstore/views/block.py @@ -1,8 +1,10 @@ """Views for blocks.""" import logging +import re from collections import OrderedDict from functools import partial +from urllib.parse import urlparse from django.contrib.auth.decorators import login_required from django.core.exceptions import PermissionDenied @@ -305,6 +307,43 @@ def xblock_view_handler(request, usage_key_string, view_name): return HttpResponse(status=406) +def _get_safe_return_to(request): + """ + Read and validate the ``returnTo`` query parameter for the XBlock edit view. + + Returns the parameter value if it is a safe same-origin URL (i.e. an + absolute-path reference that starts with ``/`` but not ``//``), or ``None`` + if the parameter is absent or fails validation. This prevents open-redirect + attacks via protocol-relative URLs such as ``//evil.com/path``. + """ + return_to = request.GET.get('returnTo', '').strip() + if not return_to: + return None + + if re.search(r'[\x00-\x1f\x7f]', return_to): + return None + + if len(return_to) > 2048: + return None + + parsed = urlparse(return_to) + if parsed.scheme or parsed.netloc: + request_origin = '{scheme}://{host}'.format( + scheme=request.scheme, + host=request.get_host(), + ) + url_origin = '{scheme}://{host}'.format( + scheme=parsed.scheme, + host=parsed.netloc, + ) + if request_origin != url_origin: + return None + elif not return_to.startswith('/') or return_to.startswith('//'): + return None + + return return_to + + @xframe_options_exempt @require_http_methods(["GET"]) @login_required @@ -313,6 +352,10 @@ def xblock_edit_view(request, usage_key_string): Return rendered xblock edit view. Allows editing of an XBlock specified by the usage key. + + Supports an optional ``returnTo`` query parameter. When present and + pointing to a same-origin URL, the editor will redirect the browser to + that URL after the user saves or cancels instead of leaving the page blank. """ usage_key = usage_key_with_run(usage_key_string) if not has_studio_read_access(request.user, usage_key.course_key): @@ -333,6 +376,7 @@ def xblock_edit_view(request, usage_key_string): container_handler_context.update({ "action_name": "edit", "resources": list(hashed_resources.items()), + "return_to": _get_safe_return_to(request), }) return render_to_response('container_editor.html', container_handler_context) diff --git a/cms/djangoapps/contentstore/views/certificate_manager.py b/cms/djangoapps/contentstore/views/certificate_manager.py index 429950477fdd..081afdcc0dd7 100644 --- a/cms/djangoapps/contentstore/views/certificate_manager.py +++ b/cms/djangoapps/contentstore/views/certificate_manager.py @@ -121,7 +121,7 @@ def is_activated(course): along with the certificates. """ is_active = False - certificates = None + certificates = [] if settings.FEATURES.get('CERTIFICATES_HTML_VIEW', False): certificates = CertificateManager.get_certificates(course) # we are assuming only one certificate in certificates collection. diff --git a/cms/djangoapps/contentstore/views/component.py b/cms/djangoapps/contentstore/views/component.py index 34c1f465c566..ea0b5876135f 100644 --- a/cms/djangoapps/contentstore/views/component.py +++ b/cms/djangoapps/contentstore/views/component.py @@ -43,6 +43,12 @@ from xmodule.modulestore.django import modulestore # lint-amnesty, pylint: disable=wrong-import-order from xmodule.modulestore.exceptions import ItemNotFoundError # lint-amnesty, pylint: disable=wrong-import-order +try: + from games.toggles import is_games_xblock_enabled # pylint: disable=import-error +except ImportError: + def is_games_xblock_enabled(): + return False + __all__ = [ 'container_handler', 'component_handler', @@ -83,13 +89,29 @@ ] DEFAULT_ADVANCED_MODULES = [ + 'annotatable', + 'done', + 'split_test', + 'freetextresponse', 'google-calendar', 'google-document', + 'imagemodal', + 'h5pxblock', + 'invideoquiz', 'lti_consumer', + 'oppia', + 'ubcpi', 'poll', - 'split_test', + 'qualtricssurvey', + 'scorm', + 'edx_sga', + 'submit-and-compare', 'survey', 'word_cloud', + 'recommender', + 'library_content', + 'schoolyourself_lesson', + 'schoolyourself_review', ] @@ -295,6 +317,11 @@ def create_support_legend_dict(): # by the components in the order listed in COMPONENT_TYPES. component_types = COMPONENT_TYPES[:] + # Add games xblock if enabled (checked at request time) + if is_games_xblock_enabled(): + component_types.append('games') + component_display_names['games'] = _("Games") + # Libraries do not support discussions, drag-and-drop, and openassessment and other libraries component_not_supported_by_library = [ 'discussion', @@ -439,12 +466,16 @@ def create_support_legend_dict(): ) categories.add(component) + beta_types = BETA_COMPONENT_TYPES[:] + if is_games_xblock_enabled() and category == 'games': + beta_types.append('games') + component_templates.append({ "type": category, "templates": templates_for_category, "display_name": component_display_names[category], "support_legend": create_support_legend_dict(), - "beta": category in BETA_COMPONENT_TYPES, + "beta": category in beta_types, }) # Libraries do not support advanced components at this time. diff --git a/cms/djangoapps/contentstore/views/course.py b/cms/djangoapps/contentstore/views/course.py index fa8769dc0cb9..681ea5f9fe9b 100644 --- a/cms/djangoapps/contentstore/views/course.py +++ b/cms/djangoapps/contentstore/views/course.py @@ -7,14 +7,19 @@ import random import re import string -from typing import Dict, NamedTuple, Optional +from typing import Dict import django.utils from ccx_keys.locator import CCXLocator from django.conf import settings from django.contrib.auth import get_user_model from django.contrib.auth.decorators import login_required -from django.core.exceptions import FieldError, PermissionDenied, ValidationError as DjangoValidationError +from django.core.exceptions import ( + FieldError, + ImproperlyConfigured, + PermissionDenied, + ValidationError as DjangoValidationError, +) from django.db.models import QuerySet from django.http import Http404, HttpResponse, HttpResponseBadRequest, HttpResponseNotFound from django.shortcuts import redirect @@ -39,6 +44,7 @@ from cms.djangoapps.models.settings.course_grading import CourseGradingModel from cms.djangoapps.models.settings.course_metadata import CourseMetadata from cms.djangoapps.models.settings.encoder import CourseSettingsEncoder +from cms.djangoapps.modulestore_migrator.data import ModulestoreMigration from cms.djangoapps.contentstore.api.views.utils import get_bool_param from common.djangoapps.course_action_state.managers import CourseActionStateItemNotFoundError from common.djangoapps.course_action_state.models import CourseRerunState, CourseRerunUIStateManager @@ -56,6 +62,7 @@ GlobalStaff, UserBasedRole, OrgStaffRole, + strict_role_checking, ) from common.djangoapps.util.json_request import JsonResponse, JsonResponseBadRequest, expect_json from common.djangoapps.util.string_utils import _has_non_ascii_characters @@ -85,8 +92,6 @@ from ..tasks import rerun_course as rerun_course_task from ..toggles import ( default_enable_flexible_peer_openassessments, - use_new_course_outline_page, - use_new_home_page, use_new_updates_page, use_new_advanced_settings_page, use_new_grading_page, @@ -98,15 +103,12 @@ add_instructor, get_advanced_settings_url, get_course_grading, - get_course_index_context, get_course_outline_url, get_course_rerun_context, get_course_settings, get_grading_url, get_group_configurations_context, get_group_configurations_url, - get_home_context, - get_library_context, get_lms_link_for_item, get_proctored_exam_settings_url, get_schedule_details_url, @@ -536,7 +538,9 @@ def filter_ccx(course_access): return not isinstance(course_access.course_id, CCXLocator) instructor_courses = UserBasedRole(request.user, CourseInstructorRole.ROLE).courses_with_role() - staff_courses = UserBasedRole(request.user, CourseStaffRole.ROLE).courses_with_role() + with strict_role_checking(): + staff_courses = UserBasedRole(request.user, CourseStaffRole.ROLE).courses_with_role() + all_courses = list(filter(filter_ccx, instructor_courses | staff_courses)) courses_list = [] course_keys = {} @@ -652,11 +656,7 @@ def course_listing(request): """ List all courses and libraries available to the logged in user """ - if use_new_home_page(): - return redirect(get_studio_home_url()) - - home_context = get_home_context(request) - return render_to_response('index.html', home_context) + return redirect(get_studio_home_url()) @login_required @@ -665,15 +665,28 @@ def library_listing(request): """ List all Libraries available to the logged in user """ - data = get_library_context(request) - return render_to_response('index.html', data) + mfe_base_url = settings.COURSE_AUTHORING_MICROFRONTEND_URL + if mfe_base_url: + return redirect(f'{mfe_base_url}/libraries') + + raise ImproperlyConfigured( + "The COURSE_AUTHORING_MICROFRONTEND_URL must be configured. " + "Please set it to the base url for your authoring MFE." + ) -def _format_library_for_view(library, request, migrated_to: Optional[NamedTuple]): +def format_library_for_view(library, request, migration: ModulestoreMigration | None): """ Return a dict of the data which the view requires for each library """ - + migration_info = {} + if migration: + migration_info = { + 'migrated_to_key': migration.target_key, + 'migrated_to_title': migration.target_title, + 'migrated_to_collection_key': migration.target_collection_slug, + 'migrated_to_collection_title': migration.target_collection_title, + } return { 'display_name': library.display_name, 'library_key': str(library.location.library_key), @@ -681,7 +694,8 @@ def _format_library_for_view(library, request, migrated_to: Optional[NamedTuple] 'org': library.display_org_with_default, 'number': library.display_number_with_default, 'can_edit': has_studio_write_access(request.user, library.location.library_key), - **(migrated_to._asdict() if migrated_to is not None else {}), + 'is_migrated': migration is not None, + **migration_info, } @@ -736,18 +750,8 @@ def course_index(request, course_key): org, course, name: Attributes of the Location for the item to edit """ - if use_new_course_outline_page(course_key): - block_to_show = request.GET.get("show") - return redirect(get_course_outline_url(course_key, block_to_show)) - with modulestore().bulk_operations(course_key): - # A depth of None implies the whole course. The course outline needs this in order to compute has_changes. - # A unit may not have a draft version, but one of its components could, and hence the unit itself has changes. - course_block = get_course_and_check_access(course_key, request.user, depth=None) - if not course_block: - raise Http404 - # should be under bulk_operations if course_block is passed - course_index_context = get_course_index_context(request, course_key, course_block) - return render_to_response('course_outline.html', course_index_context) + block_to_show = request.GET.get("show") + return redirect(get_course_outline_url(course_key, block_to_show)) @function_trace('get_courses_accessible_to_user') @@ -1848,12 +1852,20 @@ def get_allowed_organizations_for_libraries(user): """ Helper method for returning the list of organizations for which the user is allowed to create libraries. """ + organizations_set = set() + + # This allows org-level staff to create libraries. We should re-evaluate + # whether this is necessary and try to normalize course and library creation + # authorization behavior. if settings.FEATURES.get('ENABLE_ORGANIZATION_STAFF_ACCESS_FOR_CONTENT_LIBRARIES', False): - return get_organizations_for_non_course_creators(user) - elif settings.FEATURES.get('ENABLE_CREATOR_GROUP', False): - return get_organizations(user) - else: - return [] + organizations_set.update(get_organizations_for_non_course_creators(user)) + + # This allows people in the course creator group for an org to create + # libraries, which mimics course behavior. + if settings.FEATURES.get('ENABLE_CREATOR_GROUP', False): + organizations_set.update(get_organizations(user)) + + return sorted(organizations_set) def user_can_create_organizations(user): diff --git a/cms/djangoapps/contentstore/views/tests/test_assets.py b/cms/djangoapps/contentstore/views/tests/test_assets.py index 2b13338c0dc7..e7dbcfe9f55a 100644 --- a/cms/djangoapps/contentstore/views/tests/test_assets.py +++ b/cms/djangoapps/contentstore/views/tests/test_assets.py @@ -12,13 +12,11 @@ from ddt import data, ddt from django.conf import settings from django.test.utils import override_settings -from edx_toggles.toggles.testutils import override_waffle_flag from opaque_keys.edx.keys import AssetKey from opaque_keys.edx.locator import CourseLocator from PIL import Image from pytz import UTC -from cms.djangoapps.contentstore import toggles from cms.djangoapps.contentstore.tests.utils import CourseTestCase from cms.djangoapps.contentstore.utils import reverse_course_url from cms.djangoapps.contentstore.views import assets @@ -87,10 +85,9 @@ class BasicAssetsTestCase(AssetsTestCase): Test getting assets via html w/o additional args """ - @override_waffle_flag(toggles.LEGACY_STUDIO_FILES_UPLOADS, True) def test_basic(self): resp = self.client.get(self.url, HTTP_ACCEPT='text/html') - self.assertEqual(resp.status_code, 200) + self.assertEqual(resp.status_code, 302) def test_static_url_generation(self): diff --git a/cms/djangoapps/contentstore/views/tests/test_block.py b/cms/djangoapps/contentstore/views/tests/test_block.py index 01aff3d613c1..b17c71668a78 100644 --- a/cms/djangoapps/contentstore/views/tests/test_block.py +++ b/cms/djangoapps/contentstore/views/tests/test_block.py @@ -76,7 +76,8 @@ from openedx.core.djangoapps.discussions.models import DiscussionsConfiguration from openedx.core.djangoapps.content_tagging import api as tagging_api -from ..component import component_handler, DEFAULT_ADVANCED_MODULES, get_component_templates +from ..block import _get_safe_return_to +from ..component import component_handler, get_component_templates from cms.djangoapps.contentstore.xblock_storage_handlers.view_handlers import ( ALWAYS, VisibilityState, @@ -86,6 +87,7 @@ add_container_page_publishing_info, create_xblock_info, ) +from common.test.utils import assert_dict_contains_subset class AsideTest(XBlockAside): @@ -863,7 +865,8 @@ def test_duplicate_event(self): XBLOCK_DUPLICATED.connect(event_receiver) usage_key = self._duplicate_and_verify(self.vert_usage_key, self.seq_usage_key) event_receiver.assert_called() - self.assertDictContainsSubset( + assert_dict_contains_subset( + self, { "signal": XBLOCK_DUPLICATED, "sender": None, @@ -1853,7 +1856,7 @@ def setUp(self): @XBlockAside.register_temp_plugin(AsideTest, "test_aside") @patch( - "xmodule.modulestore.split_mongo.caching_descriptor_system.CachingDescriptorSystem.applicable_aside_types", + "xmodule.modulestore.split_mongo.runtime.SplitModuleStoreRuntime.applicable_aside_types", lambda self, block: ["test_aside"], ) def test_duplicate_equality_with_asides(self): @@ -2698,8 +2701,8 @@ def test_add_groups(self): group_id_to_child = split_test.group_id_to_child.copy() self.assertEqual(2, len(group_id_to_child)) - # CachingDescriptorSystem is used in tests. - # CachingDescriptorSystem doesn't have user service, that's needed for + # SplitModuleStoreRuntime is used in tests. + # SplitModuleStoreRuntime doesn't have user service, that's needed for # SplitTestBlock. So, in this line of code we add this service manually. split_test.runtime._services["user"] = DjangoXBlockUserService( # pylint: disable=protected-access self.user @@ -2974,10 +2977,23 @@ def test_basic_components(self): self.assertGreater(len(self.get_templates_of_type("html")), 0) self.assertGreater(len(self.get_templates_of_type("problem")), 0) - # Check for default advanced modules + # Check for default advanced modules - only the ones available in test environment advanced_templates = self.get_templates_of_type("advanced") advanced_module_keys = [t['category'] for t in advanced_templates] - self.assertCountEqual(advanced_module_keys, DEFAULT_ADVANCED_MODULES) + expected_advanced_modules = [ + 'annotatable', + 'done', + 'google-calendar', + 'google-document', + 'lti_consumer', + 'poll', + 'split_test', + 'survey', + 'word_cloud', + 'recommender', + 'edx_sga', + ] + self.assertCountEqual(advanced_module_keys, expected_advanced_modules) # Now fully disable video through XBlockConfiguration XBlockConfiguration.objects.create(name="video", enabled=False) @@ -3025,16 +3041,6 @@ def test_advanced_components(self): """ Test the handling of advanced component templates. """ - self.course.advanced_modules.append("done") - EXPECTED_ADVANCED_MODULES_LENGTH = len(DEFAULT_ADVANCED_MODULES) + 1 - self.templates = get_component_templates(self.course) - advanced_templates = self.get_templates_of_type("advanced") - self.assertEqual(len(advanced_templates), EXPECTED_ADVANCED_MODULES_LENGTH) - done_template = advanced_templates[0] - self.assertEqual(done_template.get("category"), "done") - self.assertEqual(done_template.get("display_name"), "Completion") - self.assertIsNone(done_template.get("boilerplate_name", None)) - # Verify that components are not added twice self.course.advanced_modules.append("video") self.course.advanced_modules.append("drag-and-drop-v2") @@ -3045,7 +3051,6 @@ def test_advanced_components(self): self.templates = get_component_templates(self.course) advanced_templates = self.get_templates_of_type("advanced") - self.assertEqual(len(advanced_templates), EXPECTED_ADVANCED_MODULES_LENGTH) only_template = advanced_templates[0] self.assertNotEqual(only_template.get("category"), "video") self.assertNotEqual(only_template.get("category"), "drag-and-drop-v2") @@ -3118,8 +3123,13 @@ def test_create_support_level_flag_off(self): """ XBlockStudioConfigurationFlag.objects.create(enabled=False) self.course.advanced_modules.extend(["annotatable", "done"]) - expected_xblocks = ["Annotation", "Completion"] + self.default_advanced_modules_titles - self._verify_advanced_xblocks(expected_xblocks, [True] * len(expected_xblocks)) + # Get actual templates to determine count dynamically + templates = get_component_templates(self.course) + advanced_templates = templates[-1]["templates"] + expected_count = len(advanced_templates) + # Verify all advanced templates have support_level=True + for template in advanced_templates: + self.assertTrue(template["support_level"]) def test_xblock_masquerading_as_problem(self): """ @@ -4447,7 +4457,7 @@ def test_self_paced_item_visibility_state(self): @patch( - "xmodule.modulestore.split_mongo.caching_descriptor_system.CachingDescriptorSystem.applicable_aside_types", + "xmodule.modulestore.split_mongo.runtime.SplitModuleStoreRuntime.applicable_aside_types", lambda self, block: ["test_aside"], ) class TestUpdateFromSource(ModuleStoreTestCase): @@ -4628,3 +4638,93 @@ def test_xblock_edit_view_contains_resources(self): self.assertGreater(len(resource_links), 0, f"No CSS resources found in HTML. Found: {resource_links}") self.assertGreater(len(script_sources), 0, f"No JS resources found in HTML. Found: {script_sources}") + + +class TestGetSafeReturnTo(TestCase): + """ + Tests for _get_safe_return_to validation. + """ + + def setUp(self): + super().setUp() + self.factory = RequestFactory() + + def _make_request(self, return_to=None): + """Build a GET request with an optional returnTo query parameter.""" + url = '/dummy' + if return_to is not None: + url = f'/dummy?returnTo={return_to}' + return self.factory.get(url) + + # -- valid inputs -------------------------------------------------------- + + def test_valid_relative_path(self): + request = self._make_request('/course/123') + self.assertEqual(_get_safe_return_to(request), '/course/123') + + def test_valid_root_path(self): + request = self._make_request('/') + self.assertEqual(_get_safe_return_to(request), '/') + + def test_valid_relative_path_with_query_string(self): + request = self.factory.get('/dummy', {'returnTo': '/course/123?tab=outline'}) + self.assertEqual(_get_safe_return_to(request), '/course/123?tab=outline') + + def test_valid_absolute_url_same_origin(self): + request = self.factory.get('/dummy', {'returnTo': 'http://testserver/course/123'}) + self.assertEqual(_get_safe_return_to(request), 'http://testserver/course/123') + + # -- empty / missing values ---------------------------------------------- + + def test_missing_parameter(self): + request = self.factory.get('/dummy') + self.assertIsNone(_get_safe_return_to(request)) + + def test_empty_string(self): + request = self._make_request('') + self.assertIsNone(_get_safe_return_to(request)) + + def test_whitespace_only(self): + request = self.factory.get('/dummy', {'returnTo': ' '}) + self.assertIsNone(_get_safe_return_to(request)) + + # -- protocol-relative / different origin -------------------------------- + + def test_protocol_relative_url_rejected(self): + request = self._make_request('//evil.com/path') + self.assertIsNone(_get_safe_return_to(request)) + + def test_absolute_url_different_origin_rejected(self): + request = self.factory.get('/dummy', {'returnTo': 'https://evil.com/steal'}) + self.assertIsNone(_get_safe_return_to(request)) + + def test_absolute_url_different_scheme_rejected(self): + request = self.factory.get('/dummy', {'returnTo': 'https://testserver/course'}) + self.assertIsNone(_get_safe_return_to(request)) + + # -- relative paths that don't start with / ------------------------------ + + def test_bare_relative_path_rejected(self): + request = self._make_request('course/123') + self.assertIsNone(_get_safe_return_to(request)) + + # -- control characters -------------------------------------------------- + + def test_null_byte_rejected(self): + request = self.factory.get('/dummy', {'returnTo': '/course/\x00'}) + self.assertIsNone(_get_safe_return_to(request)) + + def test_newline_rejected(self): + request = self.factory.get('/dummy', {'returnTo': '/course/\n/path'}) + self.assertIsNone(_get_safe_return_to(request)) + + def test_tab_character_rejected(self): + request = self.factory.get('/dummy', {'returnTo': '/course/\t/path'}) + self.assertIsNone(_get_safe_return_to(request)) + + # -- length limit -------------------------------------------------------- + + def test_url_over_2048_chars_rejected(self): + long_url = '/' + 'a' * 2048 + request = self.factory.get('/dummy', {'returnTo': long_url}) + self.assertIsNone(_get_safe_return_to(request)) diff --git a/cms/djangoapps/contentstore/views/tests/test_clipboard_paste.py b/cms/djangoapps/contentstore/views/tests/test_clipboard_paste.py index 3fae9d996fd2..2ca03ccf892b 100644 --- a/cms/djangoapps/contentstore/views/tests/test_clipboard_paste.py +++ b/cms/djangoapps/contentstore/views/tests/test_clipboard_paste.py @@ -16,7 +16,7 @@ from openedx_tagging.core.tagging.models import Tag from organizations.models import Organization from xmodule.modulestore.django import contentstore, modulestore -from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase, upload_file_to_course +from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase, upload_file_to_course, ImmediateOnCommitMixin from xmodule.modulestore.tests.factories import BlockFactory, CourseFactory, ToyCourseFactory, LibraryFactory from cms.djangoapps.contentstore.utils import reverse_usage_url @@ -400,7 +400,7 @@ def test_paste_with_assets(self): assert source_pic2_hash != dest_pic2_hash # Because there was a conflict, this file was unchanged. -class ClipboardPasteFromV2LibraryTestCase(OpenEdxEventsTestMixin, ModuleStoreTestCase): +class ClipboardPasteFromV2LibraryTestCase(OpenEdxEventsTestMixin, ImmediateOnCommitMixin, ModuleStoreTestCase): """ Test Clipboard Paste functionality with a "new" (as of Sumac) library """ diff --git a/cms/djangoapps/contentstore/views/tests/test_course_index.py b/cms/djangoapps/contentstore/views/tests/test_course_index.py index a22ce637fedd..58c425c601a3 100644 --- a/cms/djangoapps/contentstore/views/tests/test_course_index.py +++ b/cms/djangoapps/contentstore/views/tests/test_course_index.py @@ -8,496 +8,28 @@ from unittest import mock, skip import ddt -import lxml import pytz -from django.conf import settings from django.core.exceptions import PermissionDenied -from django.test.utils import override_settings from django.utils.translation import gettext as _ -from edx_toggles.toggles.testutils import override_waffle_flag -from opaque_keys.edx.locator import CourseLocator from search.api import perform_search -from cms.djangoapps.contentstore import toggles from cms.djangoapps.contentstore.courseware_index import CoursewareSearchIndexer, SearchIndexingError from cms.djangoapps.contentstore.tests.utils import CourseTestCase from cms.djangoapps.contentstore.utils import ( - add_instructor, - get_proctored_exam_settings_url, reverse_course_url, reverse_usage_url ) -from common.djangoapps.course_action_state.managers import CourseRerunUIStateManager -from common.djangoapps.course_action_state.models import CourseRerunState -from common.djangoapps.student.auth import has_course_author_access -from common.djangoapps.student.roles import CourseStaffRole, GlobalStaff, LibraryUserRole from common.djangoapps.student.tests.factories import UserFactory -from openedx.core.djangoapps.content.course_overviews.tests.factories import CourseOverviewFactory from openedx.core.djangoapps.waffle_utils.testutils import WAFFLE_TABLES from xmodule.modulestore.django import modulestore # lint-amnesty, pylint: disable=wrong-import-order from xmodule.modulestore.exceptions import ItemNotFoundError # lint-amnesty, pylint: disable=wrong-import-order from xmodule.modulestore.tests.django_utils import TEST_DATA_SPLIT_MODULESTORE -from xmodule.modulestore.tests.factories import CourseFactory, BlockFactory, LibraryFactory, check_mongo_calls # lint-amnesty, pylint: disable=wrong-import-order +from xmodule.modulestore.tests.factories import BlockFactory, check_mongo_calls # lint-amnesty, pylint: disable=wrong-import-order from ..course import _deprecated_blocks_info, course_outline_initial_state, reindex_course_and_check_access from cms.djangoapps.contentstore.xblock_storage_handlers.view_handlers import VisibilityState, create_xblock_info -@override_waffle_flag(toggles.LEGACY_STUDIO_HOME, True) -@override_waffle_flag(toggles.LEGACY_STUDIO_COURSE_OUTLINE, True) -class TestCourseIndex(CourseTestCase): - """ - Unit tests for getting the list of courses and the course outline. - """ - - MODULESTORE = TEST_DATA_SPLIT_MODULESTORE - - def setUp(self): - """ - Add a course with odd characters in the fields - """ - super().setUp() - # had a problem where index showed course but has_access failed to retrieve it for non-staff - self.odd_course = CourseFactory.create( - org='test.org_1-2', - number='test-2.3_course', - display_name='dotted.course.name-2', - ) - CourseOverviewFactory.create( - id=self.odd_course.id, - org=self.odd_course.org, - display_name=self.odd_course.display_name, - ) - - def check_courses_on_index(self, authed_client, expected_course_tab_len): - """ - Test that the React course listing is present. - """ - index_url = '/home/' - index_response = authed_client.get(index_url, {}, HTTP_ACCEPT='text/html') - parsed_html = lxml.html.fromstring(index_response.content) - courses_tab = parsed_html.find_class('react-course-listing') - self.assertEqual(len(courses_tab), expected_course_tab_len) - - def test_libraries_on_index(self): - """ - Test that the library tab is present. - """ - def _assert_library_tab_present(response): - """ - Asserts there's a library tab. - """ - parsed_html = lxml.html.fromstring(response.content) - library_tab = parsed_html.find_class('react-library-listing') - self.assertEqual(len(library_tab), 1) - - # Add a library: - lib1 = LibraryFactory.create() # lint-amnesty, pylint: disable=unused-variable - - index_url = '/home/' - index_response = self.client.get(index_url, {}, HTTP_ACCEPT='text/html') - _assert_library_tab_present(index_response) - - # Make sure libraries are visible to non-staff users too - self.client.logout() - non_staff_user, non_staff_userpassword = self.create_non_staff_user() - lib2 = LibraryFactory.create(user_id=non_staff_user.id) - LibraryUserRole(lib2.location.library_key).add_users(non_staff_user) - self.client.login(username=non_staff_user.username, password=non_staff_userpassword) - index_response = self.client.get(index_url, {}, HTTP_ACCEPT='text/html') - _assert_library_tab_present(index_response) - - def test_is_staff_access(self): - """ - Test that people with is_staff see the courses and can navigate into them - """ - self.check_courses_on_index(self.client, 1) - - def test_negative_conditions(self): - """ - Test the error conditions for the access - """ - outline_url = reverse_course_url('course_handler', self.course.id) - # register a non-staff member and try to delete the course branch - non_staff_client, _ = self.create_non_staff_authed_user_client() - response = non_staff_client.delete(outline_url, {}, HTTP_ACCEPT='application/json') - if self.course.id.deprecated: - self.assertEqual(response.status_code, 404) - else: - self.assertEqual(response.status_code, 403) - - def test_course_staff_access(self): - """ - Make and register course_staff and ensure they can access the courses - """ - course_staff_client, course_staff = self.create_non_staff_authed_user_client() - for course in [self.course, self.odd_course]: - permission_url = reverse_course_url('course_team_handler', course.id, kwargs={'email': course_staff.email}) - - self.client.post( - permission_url, - data=json.dumps({"role": "staff"}), - content_type="application/json", - HTTP_ACCEPT="application/json", - ) - - # test access - self.check_courses_on_index(course_staff_client, 1) - - def test_json_responses(self): - - outline_url = reverse_course_url('course_handler', self.course.id) - chapter = BlockFactory.create(parent_location=self.course.location, category='chapter', display_name="Week 1") - lesson = BlockFactory.create(parent_location=chapter.location, category='sequential', display_name="Lesson 1") - subsection = BlockFactory.create( - parent_location=lesson.location, - category='vertical', - display_name='Subsection 1' - ) - BlockFactory.create(parent_location=subsection.location, category="video", display_name="My Video") - - resp = self.client.get(outline_url, HTTP_ACCEPT='application/json') - - if self.course.id.deprecated: - self.assertEqual(resp.status_code, 404) - return - - json_response = json.loads(resp.content.decode('utf-8')) - - # First spot check some values in the root response - self.assertEqual(json_response['category'], 'course') - self.assertEqual(json_response['id'], str(self.course.location)) - self.assertEqual(json_response['display_name'], self.course.display_name) - self.assertTrue(json_response['published']) - self.assertIsNone(json_response['visibility_state']) - - # Now verify the first child - children = json_response['child_info']['children'] - self.assertGreater(len(children), 0) - first_child_response = children[0] - self.assertEqual(first_child_response['category'], 'chapter') - self.assertEqual(first_child_response['id'], str(chapter.location)) - self.assertEqual(first_child_response['display_name'], 'Week 1') - self.assertTrue(json_response['published']) - self.assertEqual(first_child_response['visibility_state'], VisibilityState.unscheduled) - self.assertGreater(len(first_child_response['child_info']['children']), 0) - - # Finally, validate the entire response for consistency - self.assert_correct_json_response(json_response) - - def test_notifications_handler_get(self): - state = CourseRerunUIStateManager.State.FAILED - action = CourseRerunUIStateManager.ACTION - should_display = True - - # try when no notification exists - notification_url = reverse_course_url('course_notifications_handler', self.course.id, kwargs={ - 'action_state_id': 1, - }) - - resp = self.client.get(notification_url, HTTP_ACCEPT='application/json') - - # verify that we get an empty dict out - self.assertEqual(resp.status_code, 400) - - # create a test notification - rerun_state = CourseRerunState.objects.update_state( - course_key=self.course.id, - new_state=state, - allow_not_found=True - ) - CourseRerunState.objects.update_should_display( - entry_id=rerun_state.id, - user=UserFactory(), - should_display=should_display - ) - - # try to get information on this notification - notification_url = reverse_course_url('course_notifications_handler', self.course.id, kwargs={ - 'action_state_id': rerun_state.id, - }) - resp = self.client.get(notification_url, HTTP_ACCEPT='application/json') - - json_response = json.loads(resp.content.decode('utf-8')) - - self.assertEqual(json_response['state'], state) - self.assertEqual(json_response['action'], action) - self.assertEqual(json_response['should_display'], should_display) - - def test_notifications_handler_dismiss(self): - state = CourseRerunUIStateManager.State.FAILED - should_display = True - rerun_course_key = CourseLocator(org='testx', course='test_course', run='test_run') - - # add an instructor to this course - user2 = UserFactory() - add_instructor(rerun_course_key, self.user, user2) - - # create a test notification - rerun_state = CourseRerunState.objects.update_state( - course_key=rerun_course_key, - new_state=state, - allow_not_found=True - ) - CourseRerunState.objects.update_should_display( - entry_id=rerun_state.id, - user=user2, - should_display=should_display - ) - - # try to get information on this notification - notification_dismiss_url = reverse_course_url('course_notifications_handler', self.course.id, kwargs={ - 'action_state_id': rerun_state.id, - }) - resp = self.client.delete(notification_dismiss_url) - self.assertEqual(resp.status_code, 200) - - with self.assertRaises(CourseRerunState.DoesNotExist): - # delete nofications that are dismissed - CourseRerunState.objects.get(id=rerun_state.id) - - self.assertFalse(has_course_author_access(user2, rerun_course_key)) - - def assert_correct_json_response(self, json_response): - """ - Asserts that the JSON response is syntactically consistent - """ - self.assertIsNotNone(json_response['display_name']) - self.assertIsNotNone(json_response['id']) - self.assertIsNotNone(json_response['category']) - self.assertTrue(json_response['published']) - if json_response.get('child_info', None): - for child_response in json_response['child_info']['children']: - self.assert_correct_json_response(child_response) - - def test_course_updates_invalid_url(self): - """ - Tests the error conditions for the invalid course updates URL. - """ - # Testing the response code by passing slash separated course id whose format is valid but no course - # having this id exists. - invalid_course_key = f'{self.course.id}_blah_blah_blah' - course_updates_url = reverse_course_url('course_info_handler', invalid_course_key) - response = self.client.get(course_updates_url) - self.assertEqual(response.status_code, 404) - - # Testing the response code by passing split course id whose format is valid but no course - # having this id exists. - split_course_key = CourseLocator(org='orgASD', course='course_01213', run='Run_0_hhh_hhh_hhh') - course_updates_url_split = reverse_course_url('course_info_handler', split_course_key) - response = self.client.get(course_updates_url_split) - self.assertEqual(response.status_code, 404) - - # Testing the response by passing split course id whose format is invalid. - invalid_course_id = f'invalid.course.key/{split_course_key}' - course_updates_url_split = reverse_course_url('course_info_handler', invalid_course_id) - response = self.client.get(course_updates_url_split) - self.assertEqual(response.status_code, 404) - - def test_course_index_invalid_url(self): - """ - Tests the error conditions for the invalid course index URL. - """ - # Testing the response code by passing slash separated course key, no course - # having this key exists. - invalid_course_key = f'{self.course.id}_some_invalid_run' - course_outline_url = reverse_course_url('course_handler', invalid_course_key) - response = self.client.get_html(course_outline_url) - self.assertEqual(response.status_code, 404) - - # Testing the response code by passing split course key, no course - # having this key exists. - split_course_key = CourseLocator(org='invalid_org', course='course_01111', run='Run_0_invalid') - course_outline_url_split = reverse_course_url('course_handler', split_course_key) - response = self.client.get_html(course_outline_url_split) - self.assertEqual(response.status_code, 404) - - def test_course_outline_with_display_course_number_as_none(self): - """ - Tests course outline when 'display_coursenumber' field is none. - """ - # Change 'display_coursenumber' field to None and update the course. - self.course.display_coursenumber = None - updated_course = self.update_course(self.course, self.user.id) - - # Assert that 'display_coursenumber' field has been changed successfully. - self.assertEqual(updated_course.display_coursenumber, None) - - # Perform GET request on course outline url with the course id. - course_outline_url = reverse_course_url('course_handler', updated_course.id) - response = self.client.get_html(course_outline_url) - - # course_handler raise 404 for old mongo course - if self.course.id.deprecated: - self.assertEqual(response.status_code, 404) - return - - # Assert that response code is 200. - self.assertEqual(response.status_code, 200) - - # Assert that 'display_course_number' is being set to "" (as display_coursenumber was None). - self.assertContains(response, 'display_course_number: ""') - - -@override_waffle_flag(toggles.LEGACY_STUDIO_HOME, True) -@ddt.ddt -class TestCourseIndexArchived(CourseTestCase): - """ - Unit tests for testing the course index list when there are archived courses. - """ - - MODULESTORE = TEST_DATA_SPLIT_MODULESTORE - - NOW = datetime.datetime.now(pytz.utc) - DAY = datetime.timedelta(days=1) - YESTERDAY = NOW - DAY - TOMORROW = NOW + DAY - - ORG = 'MyOrg' - - ENABLE_SEPARATE_ARCHIVED_COURSES = settings.FEATURES.copy() - ENABLE_SEPARATE_ARCHIVED_COURSES['ENABLE_SEPARATE_ARCHIVED_COURSES'] = True - DISABLE_SEPARATE_ARCHIVED_COURSES = settings.FEATURES.copy() - DISABLE_SEPARATE_ARCHIVED_COURSES['ENABLE_SEPARATE_ARCHIVED_COURSES'] = False - - def setUp(self): - """ - Add courses with the end date set to various values - """ - super().setUp() - - # Base course has no end date (so is active) - self.course.end = None - self.course.display_name = 'Active Course 1' - self.ORG = self.course.location.org - self.save_course() - CourseOverviewFactory.create(id=self.course.id, org=self.ORG) - - # Active course has end date set to tomorrow - self.active_course = CourseFactory.create( - display_name='Active Course 2', - org=self.ORG, - end=self.TOMORROW, - ) - CourseOverviewFactory.create( - id=self.active_course.id, - org=self.ORG, - end=self.TOMORROW, - ) - - # Archived course has end date set to yesterday - self.archived_course = CourseFactory.create( - display_name='Archived Course', - org=self.ORG, - end=self.YESTERDAY, - ) - CourseOverviewFactory.create( - id=self.archived_course.id, - org=self.ORG, - end=self.YESTERDAY, - ) - - # Base user has global staff access - self.assertTrue(GlobalStaff().has_user(self.user)) - - # Staff user just has course staff access - self.staff, self.staff_password = self.create_non_staff_user() - for course in (self.course, self.active_course, self.archived_course): - CourseStaffRole(course.id).add_users(self.staff) - - def check_index_page_with_query_count(self, separate_archived_courses, org, mongo_queries, sql_queries): - """ - Checks the index page, and ensures the number of database queries is as expected. - """ - with self.assertNumQueries(sql_queries, table_ignorelist=WAFFLE_TABLES): - with check_mongo_calls(mongo_queries): - self.check_index_page(separate_archived_courses=separate_archived_courses, org=org) - - def check_index_page(self, separate_archived_courses, org): - """ - Ensure that the index page displays the archived courses as expected. - """ - index_url = '/home/' - index_params = {} - if org is not None: - index_params['org'] = org - index_response = self.client.get(index_url, index_params, HTTP_ACCEPT='text/html') - self.assertEqual(index_response.status_code, 200) - - parsed_html = lxml.html.fromstring(index_response.content) - course_tab = parsed_html.find_class('courses') - self.assertEqual(len(course_tab), 1) - archived_course_tab = parsed_html.find_class('archived-courses') - self.assertEqual(len(archived_course_tab), 1 if separate_archived_courses else 0) - - @ddt.data( - # Staff user has course staff access - (True, 'staff', None, 23), - (False, 'staff', None, 23), - # Base user has global staff access - (True, 'user', ORG, 23), - (False, 'user', ORG, 23), - (True, 'user', None, 23), - (False, 'user', None, 23), - ) - @ddt.unpack - def test_separate_archived_courses(self, separate_archived_courses, username, org, sql_queries): - """ - Ensure that archived courses are shown as expected for all user types, when the feature is enabled/disabled. - Also ensure that enabling the feature does not adversely affect the database query count. - """ - # Authenticate the requested user - user = getattr(self, username) - password = getattr(self, username + '_password') - self.client.login(username=user, password=password) - - # Enable/disable the feature before viewing the index page. - features = settings.FEATURES.copy() - features['ENABLE_SEPARATE_ARCHIVED_COURSES'] = separate_archived_courses - with override_settings(FEATURES=features): - self.check_index_page_with_query_count(separate_archived_courses=separate_archived_courses, - org=org, - mongo_queries=0, - sql_queries=sql_queries) - - @ddt.data( - # Staff user has course staff access - (True, 'staff', None, 23), - (False, 'staff', None, 23), - # Base user has global staff access - (True, 'user', ORG, 23), - (False, 'user', ORG, 23), - (True, 'user', None, 23), - (False, 'user', None, 23), - ) - @ddt.unpack - def test_separate_archived_courses_with_home_page_course_v2_api( - self, - separate_archived_courses, - username, - org, - sql_queries - ): - """ - Ensure that archived courses are shown as expected for all user types, when the feature is enabled/disabled. - Also ensure that enabling the feature does not adversely affect the database query count. - """ - # Authenticate the requested user - user = getattr(self, username) - password = getattr(self, username + '_password') - self.client.login(username=user, password=password) - - # Enable/disable the feature before viewing the index page. - features = settings.FEATURES.copy() - features['ENABLE_SEPARATE_ARCHIVED_COURSES'] = separate_archived_courses - with override_settings(FEATURES=features): - self.check_index_page_with_query_count(separate_archived_courses=separate_archived_courses, - org=org, - mongo_queries=0, - sql_queries=sql_queries) - - -@override_waffle_flag(toggles.LEGACY_STUDIO_COURSE_OUTLINE, True) @ddt.ddt class TestCourseOutline(CourseTestCase): """ @@ -689,38 +221,15 @@ def test_verify_warn_only_on_enabled_blocks(self, enabled_block_types, deprecate expected_block_types ) - @override_settings(FEATURES={'ENABLE_EXAM_SETTINGS_HTML_VIEW': True}) - @mock.patch('cms.djangoapps.models.settings.course_metadata.CourseMetadata.validate_proctoring_settings') - def test_proctoring_link_is_visible(self, mock_validate_proctoring_settings): - """ - Test to check proctored exam settings mfe url is rendering properly - """ - mock_validate_proctoring_settings.return_value = [ - { - 'key': 'proctoring_provider', - 'message': 'error message', - 'model': {'display_name': 'proctoring_provider'} - }, - { - 'key': 'proctoring_provider', - 'message': 'error message', - 'model': {'display_name': 'proctoring_provider'} - } - ] - response = self.client.get_html(reverse_course_url('course_handler', self.course.id)) - proctored_exam_settings_url = get_proctored_exam_settings_url(self.course.id) - self.assertContains(response, proctored_exam_settings_url, 2) - def test_number_of_calls_to_db(self): """ Test to check number of queries made to mysql and mongo """ - with self.assertNumQueries(39, table_ignorelist=WAFFLE_TABLES): + with self.assertNumQueries(21, table_ignorelist=WAFFLE_TABLES): with check_mongo_calls(3): - self.client.get_html(reverse_course_url('course_handler', self.course.id)) + self.client.get(reverse_course_url('course_handler', self.course.id), content_type="application/json") -@override_waffle_flag(toggles.LEGACY_STUDIO_COURSE_OUTLINE, True) class TestCourseReIndex(CourseTestCase): """ Unit tests for the course outline. diff --git a/cms/djangoapps/contentstore/views/tests/test_exam_settings_view.py b/cms/djangoapps/contentstore/views/tests/test_exam_settings_view.py index 9bad2c77fc1a..88c4aa27eb06 100644 --- a/cms/djangoapps/contentstore/views/tests/test_exam_settings_view.py +++ b/cms/djangoapps/contentstore/views/tests/test_exam_settings_view.py @@ -24,7 +24,6 @@ "ENABLE_PROCTORED_EXAMS": True, }, ) -@override_waffle_flag(toggles.LEGACY_STUDIO_COURSE_OUTLINE, True) @override_waffle_flag(toggles.LEGACY_STUDIO_CERTIFICATES, True) @override_waffle_flag(toggles.LEGACY_STUDIO_SCHEDULE_DETAILS, True) @override_waffle_flag(toggles.LEGACY_STUDIO_CONFIGURATIONS, True) @@ -93,7 +92,6 @@ def test_view_with_exam_settings_enabled(self, handler): ) @ddt.data( "advanced_settings_handler", - "course_handler", ) def test_exam_settings_alert_with_exam_settings_enabled(self, page_handler): """ @@ -130,7 +128,6 @@ def test_exam_settings_alert_with_exam_settings_enabled(self, page_handler): ) @ddt.data( "advanced_settings_handler", - "course_handler", ) @override_waffle_flag(toggles.LEGACY_STUDIO_EXAM_SETTINGS, True) def test_exam_settings_alert_with_exam_settings_disabled(self, page_handler): @@ -173,7 +170,6 @@ def test_exam_settings_alert_with_exam_settings_disabled(self, page_handler): ) @ddt.data( "advanced_settings_handler", - "course_handler", ) def test_invalid_provider_alert(self, page_handler): """ @@ -198,7 +194,6 @@ def test_invalid_provider_alert(self, page_handler): @ddt.data( "advanced_settings_handler", - "course_handler", ) def test_exam_settings_alert_not_shown(self, page_handler): """ diff --git a/cms/djangoapps/contentstore/views/tests/test_header_menu.py b/cms/djangoapps/contentstore/views/tests/test_header_menu.py deleted file mode 100644 index fb961cc4fa89..000000000000 --- a/cms/djangoapps/contentstore/views/tests/test_header_menu.py +++ /dev/null @@ -1,93 +0,0 @@ -""" -Course Header Menu Tests. -""" -from unittest import SkipTest - -from django.conf import settings -from django.test.utils import override_settings -from edx_toggles.toggles.testutils import override_waffle_flag - -from cms.djangoapps.contentstore import toggles -from cms.djangoapps.contentstore.tests.utils import CourseTestCase -from cms.djangoapps.contentstore.utils import reverse_course_url -from common.djangoapps.util.testing import UrlResetMixin - -FEATURES_WITH_CERTS_ENABLED = settings.FEATURES.copy() -FEATURES_WITH_CERTS_ENABLED['CERTIFICATES_HTML_VIEW'] = True - -FEATURES_WITH_EXAM_SETTINGS_ENABLED = settings.FEATURES.copy() -FEATURES_WITH_EXAM_SETTINGS_ENABLED['ENABLE_EXAM_SETTINGS_HTML_VIEW'] = True - -FEATURES_WITH_EXAM_SETTINGS_DISABLED = settings.FEATURES.copy() -FEATURES_WITH_EXAM_SETTINGS_DISABLED['ENABLE_EXAM_SETTINGS_HTML_VIEW'] = False - - -@override_settings(FEATURES=FEATURES_WITH_CERTS_ENABLED) -@override_waffle_flag(toggles.LEGACY_STUDIO_COURSE_OUTLINE, True) -class TestHeaderMenu(CourseTestCase, UrlResetMixin): - """ - Unit tests for the course header menu. - """ - def setUp(self): - """ - Set up the for the course header menu tests. - """ - super().setUp() - self.reset_urls() - - def test_header_menu_without_web_certs_enabled(self): - """ - Tests course header menu should not have `Certificates` menu item - if course has not web/HTML certificates enabled. - """ - # course_handler raise 404 for old mongo course - if self.course.id.deprecated: - raise SkipTest("course_handler raise 404 for old mongo course") - self.course.cert_html_view_enabled = False - self.save_course() - outline_url = reverse_course_url('course_handler', self.course.id) - resp = self.client.get(outline_url, HTTP_ACCEPT='text/html') - self.assertEqual(resp.status_code, 200) - self.assertNotContains(resp, '')
-
- def test_create(self):
- """
- Test create announcement view
- """
- url = reverse("maintenance:announcement_create")
- self.client.post(url, {"content": "Test Create Announcement", "active": True})
- result = Announcement.objects.filter(content="Test Create Announcement").exists()
- self.assertTrue(result)
-
- def test_edit(self):
- """
- Test edit announcement view
- """
- announcement = Announcement.objects.create(content="test")
- announcement.save()
- url = reverse("maintenance:announcement_edit", kwargs={"pk": announcement.pk})
- response = self.client.get(url)
- self.assertContains(response, '
diff --git a/cms/templates/container_chromeless.html b/cms/templates/container_chromeless.html
index 647a68f32231..3ef229e906e3 100644
--- a/cms/templates/container_chromeless.html
+++ b/cms/templates/container_chromeless.html
@@ -98,13 +98,6 @@
<%static:include path="common/templates/image-modal.underscore" />
- ## The following stylesheets are included for studio-frontend debugging.
- ## Remove this as part of studio frontend deprecation.
- ## https://github.com/openedx/studio-frontend/issues/381
- % if not settings.STUDIO_FRONTEND_CONTAINER_URL:
-
-
- % endif
diff --git a/cms/templates/container_editor.html b/cms/templates/container_editor.html
index e7585d7b9664..5f00d0f61338 100644
--- a/cms/templates/container_editor.html
+++ b/cms/templates/container_editor.html
@@ -77,10 +77,6 @@
<%static:include path="common/templates/image-modal.underscore" />
- % if not settings.STUDIO_FRONTEND_CONTAINER_URL:
-
-
- % endif
% for _, resource in resources:
% if resource['kind'] == 'url' and resource['mimetype'] == 'text/css':
@@ -119,12 +115,13 @@
function (XBlockInfo, EditXBlockModal) {
var decodedActionName = '${action_name|n, decode.utf8}';
var encodedXBlockDetails = ${xblock_info | n, dump_js_escaped_json};
+ var returnTo = '${return_to or "" | n, js_escaped_string}';
if (decodedActionName === 'edit') {
var editXBlockModal = new EditXBlockModal();
var xblockInfoInstance = new XBlockInfo(encodedXBlockDetails);
- editXBlockModal.edit([], xblockInfoInstance, {});
+ editXBlockModal.edit([], xblockInfoInstance, {returnTo: returnTo || null});
}
});
diff --git a/cms/templates/course_outline.html b/cms/templates/course_outline.html
deleted file mode 100644
index 61f524123b48..000000000000
--- a/cms/templates/course_outline.html
+++ /dev/null
@@ -1,332 +0,0 @@
-<%page expression_filter="h"/>
-<%inherit file="base.html" />
-<%def name="online_help_token()"><% return "develop_course" %>
-<%!
-import logging
-from six.moves.urllib.parse import quote
-
-from cms.djangoapps.contentstore.config.waffle_utils import should_show_checklists_quality
-from common.djangoapps.util.date_utils import get_default_time_display
-from django.utils.translation import gettext as _
-from openedx.core.djangolib.js_utils import js_escaped_string, dump_js_escaped_json
-from openedx.core.djangolib.markup import HTML, Text
-from django.urls import reverse
-%>
-<%block name="title">${_("Course Outline")}
-<%block name="bodyclass">is-signedin course view-outline
-
-<%namespace name='static' file='static_content.html'/>
-
-<%block name="requirejs">
- require(["js/factories/outline"], function (OutlineFactory) {
- OutlineFactory(
- ${course_structure | n, dump_js_escaped_json},
- ${initial_state | n, dump_js_escaped_json},
- ${initial_user_clipboard | n, dump_js_escaped_json}
- );
- });
-
-
-<%block name="header_extras">
-
-% for template_name in ['course-outline', 'xblock-string-field-editor', 'basic-modal', 'modal-button', 'course-outline-modal', 'due-date-editor', 'self-paced-due-date-editor', 'release-date-editor', 'grading-editor', 'publish-editor', 'staff-lock-editor', 'unit-access-editor', 'discussion-editor', 'content-visibility-editor', 'verification-access-editor', 'timed-examination-preference-editor', 'access-editor', 'settings-modal-tabs', 'show-correctness-editor', 'highlights-editor', 'highlights-enable-editor', 'course-highlights-enable', 'course-manage-tags', 'course-video-sharing-enable', 'summary-configuration-editor', 'tag-count', 'subsection-share-link-modal-tabs', 'full-page-share-link-editor', 'embed-link-share-link-editor']:
-
-% endfor
-<%static:optional_include_mako file="course_outline_header_extras_post.html" />
-
-% if not settings.STUDIO_FRONTEND_CONTAINER_URL:
-
-
-% endif
-
-
-<%block name="page_alert">
- %if notification_dismiss_url is not None:
- ')
- self.client.post(url, {"content": "Test Edit Announcement", "active": True})
- announcement = Announcement.objects.get(pk=announcement.pk)
- self.assertEqual(announcement.content, "Test Edit Announcement")
-
- def test_delete(self):
- """
- Test delete announcement view
- """
- announcement = Announcement.objects.create(content="Test Delete")
- announcement.save()
- url = reverse("maintenance:announcement_delete", kwargs={"pk": announcement.pk})
- self.client.post(url)
- result = Announcement.objects.filter(content="Test Edit Announcement").exists()
- self.assertFalse(result)
-
- def _test_403(self, viewname, kwargs=None):
- url = reverse("maintenance:%s" % viewname, kwargs=kwargs)
- response = self.client.get(url)
- self.assertEqual(response.status_code, 403)
-
- def test_authorization(self):
- self.client.login(username=self.non_staff_user, password=self.TEST_PASSWORD)
- announcement = Announcement.objects.create(content="Test Delete")
- announcement.save()
-
- self._test_403("announcement_index")
- self._test_403("announcement_create")
- self._test_403("announcement_edit", {"pk": announcement.pk})
- self._test_403("announcement_delete", {"pk": announcement.pk})
diff --git a/cms/djangoapps/maintenance/urls.py b/cms/djangoapps/maintenance/urls.py
deleted file mode 100644
index 2937727bfa39..000000000000
--- a/cms/djangoapps/maintenance/urls.py
+++ /dev/null
@@ -1,23 +0,0 @@
-"""
-URLs for the maintenance app.
-"""
-
-from django.urls import path, re_path
-
-from .views import (
- AnnouncementCreateView,
- AnnouncementDeleteView,
- AnnouncementEditView,
- AnnouncementIndexView,
- MaintenanceIndexView
-)
-
-app_name = 'cms.djangoapps.maintenance'
-
-urlpatterns = [
- path('', MaintenanceIndexView.as_view(), name='maintenance_index'),
- re_path(r'^announcements/(?P\d+)?$', AnnouncementIndexView.as_view(), name='announcement_index'),
- path('announcements/create', AnnouncementCreateView.as_view(), name='announcement_create'),
- re_path(r'^announcements/edit/(?P\d+)?$', AnnouncementEditView.as_view(), name='announcement_edit'),
- path('announcements/delete/', AnnouncementDeleteView.as_view(), name='announcement_delete'),
-]
diff --git a/cms/djangoapps/maintenance/views.py b/cms/djangoapps/maintenance/views.py
deleted file mode 100644
index 1ec0f9c5cff2..000000000000
--- a/cms/djangoapps/maintenance/views.py
+++ /dev/null
@@ -1,190 +0,0 @@
-"""
-Views for the maintenance app.
-"""
-
-
-import logging
-
-from django.core.validators import ValidationError
-from django.urls import reverse, reverse_lazy
-from django.utils.decorators import method_decorator
-from django.utils.translation import gettext as _
-from django.views.generic import View
-from django.views.generic.edit import CreateView, DeleteView, UpdateView
-from django.views.generic.list import ListView
-from opaque_keys.edx.keys import CourseKey
-
-from common.djangoapps.edxmako.shortcuts import render_to_response
-from common.djangoapps.util.json_request import JsonResponse
-from common.djangoapps.util.views import require_global_staff
-from openedx.features.announcements.forms import AnnouncementForm
-from openedx.features.announcements.models import Announcement
-from xmodule.modulestore import ModuleStoreEnum # lint-amnesty, pylint: disable=wrong-import-order
-from xmodule.modulestore.django import modulestore # lint-amnesty, pylint: disable=wrong-import-order
-from xmodule.modulestore.exceptions import ItemNotFoundError # lint-amnesty, pylint: disable=wrong-import-order
-
-log = logging.getLogger(__name__)
-
-# This dict maintains all the views that will be used Maintenance app.
-MAINTENANCE_VIEWS = {
- 'announcement_index': {
- 'url': 'maintenance:announcement_index',
- 'name': _('Edit Announcements'),
- 'slug': 'announcement_index',
- 'description': _(
- 'This view shows the announcement editor to create or alter announcements that are shown on the right'
- 'side of the dashboard.'
- ),
- },
-}
-
-
-COURSE_KEY_ERROR_MESSAGES = {
- 'empty_course_key': _('Please provide course id.'),
- 'invalid_course_key': _('Invalid course key.'),
- 'course_key_not_found': _('No matching course found.')
-}
-
-
-class MaintenanceIndexView(View):
- """
- Index view for maintenance dashboard, used by global staff.
-
- This view lists some commands/tasks that can be used to dry run or execute directly.
- """
-
- @method_decorator(require_global_staff)
- def get(self, request):
- """Render the maintenance index view. """
- return render_to_response('maintenance/index.html', {
- 'views': MAINTENANCE_VIEWS,
- })
-
-
-class MaintenanceBaseView(View):
- """
- Base class for Maintenance views.
- """
-
- template = 'maintenance/container.html'
-
- def __init__(self, view=None):
- super().__init__()
- self.context = {
- 'view': view if view else '',
- 'form_data': {},
- 'error': False,
- 'msg': ''
- }
-
- def render_response(self):
- """
- A short method to render_to_response that renders response.
- """
- if self.request.headers.get('x-requested-with') == 'XMLHttpRequest':
- return JsonResponse(self.context)
- return render_to_response(self.template, self.context)
-
- @method_decorator(require_global_staff)
- def get(self, request):
- """
- Render get view.
- """
- return self.render_response()
-
- def validate_course_key(self, course_key, branch=ModuleStoreEnum.BranchName.draft):
- """
- Validates the course_key that would be used by maintenance app views.
-
- Arguments:
- course_key (string): a course key
- branch: a course locator branch, default value is ModuleStoreEnum.BranchName.draft .
- values can be either ModuleStoreEnum.BranchName.draft or ModuleStoreEnum.BranchName.published.
-
- Returns:
- course_usage_key (CourseLocator): course usage locator
- """
- if not course_key:
- raise ValidationError(COURSE_KEY_ERROR_MESSAGES['empty_course_key'])
-
- course_usage_key = CourseKey.from_string(course_key)
-
- if not modulestore().has_course(course_usage_key):
- raise ItemNotFoundError(COURSE_KEY_ERROR_MESSAGES['course_key_not_found'])
-
- # get branch specific locator
- course_usage_key = course_usage_key.for_branch(branch)
-
- return course_usage_key
-
-
-class AnnouncementBaseView(View):
- """
- Base view for Announcements pages
- """
-
- @method_decorator(require_global_staff)
- def dispatch(self, request, *args, **kwargs):
- return super().dispatch(request, *args, **kwargs)
-
-
-class AnnouncementIndexView(ListView, MaintenanceBaseView):
- """
- View for viewing the announcements shown on the dashboard, used by the global staff.
- """
- model = Announcement
- object_list = Announcement.objects.order_by('-active')
- context_object_name = 'announcement_list'
- paginate_by = 8
-
- def __init__(self):
- super().__init__(MAINTENANCE_VIEWS['announcement_index'])
-
- def get_context_data(self, **kwargs):
- context = super().get_context_data(**kwargs)
- context['view'] = MAINTENANCE_VIEWS['announcement_index']
- return context
-
- @method_decorator(require_global_staff)
- def get(self, request, *args, **kwargs):
- context = self.get_context_data()
- return render_to_response(self.template, context)
-
-
-class AnnouncementEditView(UpdateView, AnnouncementBaseView):
- """
- View for editing an announcement.
- """
- model = Announcement
- form_class = AnnouncementForm
- success_url = reverse_lazy('maintenance:announcement_index')
- template_name = '/maintenance/_announcement_edit.html'
-
- def get_context_data(self, **kwargs):
- context = super().get_context_data(**kwargs)
- context['action_url'] = reverse('maintenance:announcement_edit', kwargs={'pk': context['announcement'].pk})
- return context
-
-
-class AnnouncementCreateView(CreateView, AnnouncementBaseView):
- """
- View for creating an announcement.
- """
- model = Announcement
- form_class = AnnouncementForm
- success_url = reverse_lazy('maintenance:announcement_index')
- template_name = '/maintenance/_announcement_edit.html'
-
- def get_context_data(self, **kwargs):
- context = super().get_context_data(**kwargs)
- context['action_url'] = reverse('maintenance:announcement_create')
- return context
-
-
-class AnnouncementDeleteView(DeleteView, AnnouncementBaseView):
- """
- View for deleting an announcement.
- """
- model = Announcement
- success_url = reverse_lazy('maintenance:announcement_index')
- template_name = '/maintenance/_announcement_delete.html'
diff --git a/cms/djangoapps/modulestore_migrator/admin.py b/cms/djangoapps/modulestore_migrator/admin.py
index 8eef778531ac..c9a5c90256fd 100644
--- a/cms/djangoapps/modulestore_migrator/admin.py
+++ b/cms/djangoapps/modulestore_migrator/admin.py
@@ -147,8 +147,8 @@ def start_migration_task(
source_key=source.key,
target_library_key=target_library_key,
target_collection_slug=target_collection_slug,
- composition_level=form.cleaned_data['composition_level'],
- repeat_handling_strategy=form.cleaned_data['repeat_handling_strategy'],
+ composition_level=CompositionLevel(form.cleaned_data['composition_level']),
+ repeat_handling_strategy=RepeatHandlingStrategy(form.cleaned_data['repeat_handling_strategy']),
preserve_url_slugs=form.cleaned_data['preserve_url_slugs'],
forward_source_to_target=form.cleaned_data['forward_to_target'],
)
@@ -178,6 +178,7 @@ class ModulestoreBlockMigrationInline(admin.TabularInline):
"source",
"target",
"change_log_record",
+ "unsupported_reason",
)
list_display = ("id", *readonly_fields)
diff --git a/cms/djangoapps/modulestore_migrator/api.py b/cms/djangoapps/modulestore_migrator/api.py
deleted file mode 100644
index 25c26b007923..000000000000
--- a/cms/djangoapps/modulestore_migrator/api.py
+++ /dev/null
@@ -1,85 +0,0 @@
-"""
-API for migration from modulestore to learning core
-"""
-from celery.result import AsyncResult
-from opaque_keys.edx.keys import CourseKey, LearningContextKey
-from opaque_keys.edx.locator import LibraryLocator, LibraryLocatorV2
-from openedx_learning.api.authoring import get_collection
-from user_tasks.models import UserTaskStatus
-
-from openedx.core.djangoapps.content_libraries.api import get_library
-from openedx.core.types.user import AuthUser
-
-from . import tasks
-from .models import ModulestoreSource
-
-__all__ = (
- "start_migration_to_library",
- "is_successfully_migrated",
- "get_migration_info",
-)
-
-
-def start_migration_to_library(
- *,
- user: AuthUser,
- source_key: LearningContextKey,
- target_library_key: LibraryLocatorV2,
- target_collection_slug: str | None = None,
- composition_level: str,
- repeat_handling_strategy: str,
- preserve_url_slugs: bool,
- forward_source_to_target: bool,
-) -> AsyncResult:
- """
- Import a course or legacy library into a V2 library (or, a collection within a V2 library).
- """
- source, _ = ModulestoreSource.objects.get_or_create(key=source_key)
- target_library = get_library(target_library_key)
- # get_library ensures that the library is connected to a learning package.
- target_package_id: int = target_library.learning_package_id # type: ignore[assignment]
- target_collection_id = None
-
- if target_collection_slug:
- target_collection_id = get_collection(target_package_id, target_collection_slug).id
-
- return tasks.migrate_from_modulestore.delay(
- user_id=user.id,
- source_pk=source.id,
- target_package_pk=target_package_id,
- target_library_key=str(target_library_key),
- target_collection_pk=target_collection_id,
- composition_level=composition_level,
- repeat_handling_strategy=repeat_handling_strategy,
- preserve_url_slugs=preserve_url_slugs,
- forward_source_to_target=forward_source_to_target,
- )
-
-
-def is_successfully_migrated(source_key: CourseKey | LibraryLocator) -> bool:
- """
- Check if the source course/library has been migrated successfully.
- """
- return ModulestoreSource.objects.get_or_create(key=str(source_key))[0].migrations.filter(
- task_status__state=UserTaskStatus.SUCCEEDED
- ).exists()
-
-
-def get_migration_info(source_keys: list[CourseKey | LibraryLocator]) -> dict:
- """
- Check if the source course/library has been migrated successfully and return target info
- """
- return {
- info.key: info
- for info in ModulestoreSource.objects.filter(
- migrations__task_status__state=UserTaskStatus.SUCCEEDED, key__in=source_keys
- )
- .values_list(
- 'migrations__target__key',
- 'migrations__target__title',
- 'migrations__target_collection__key',
- 'migrations__target_collection__title',
- 'key',
- named=True,
- )
- }
diff --git a/cms/djangoapps/modulestore_migrator/api/__init__.py b/cms/djangoapps/modulestore_migrator/api/__init__.py
new file mode 100644
index 000000000000..f480d92bf092
--- /dev/null
+++ b/cms/djangoapps/modulestore_migrator/api/__init__.py
@@ -0,0 +1,9 @@
+
+"""
+This is the public API for the modulestore_migrator.
+"""
+
+# These wildcard imports are okay because these api modules declare __all__.
+# pylint: disable=wildcard-import
+from .read_api import *
+from .write_api import *
diff --git a/cms/djangoapps/modulestore_migrator/api/read_api.py b/cms/djangoapps/modulestore_migrator/api/read_api.py
new file mode 100644
index 000000000000..893ef416d091
--- /dev/null
+++ b/cms/djangoapps/modulestore_migrator/api/read_api.py
@@ -0,0 +1,244 @@
+"""
+API for reading information about previous migrations
+"""
+from __future__ import annotations
+
+import typing as t
+from uuid import UUID
+
+from opaque_keys.edx.keys import UsageKey
+from opaque_keys.edx.locator import (
+ LibraryLocatorV2, LibraryUsageLocatorV2, LibraryContainerLocator
+)
+from openedx_learning.api.authoring import get_draft_version
+from openedx_learning.api.authoring_models import (
+ PublishableEntityVersion, PublishableEntity, DraftChangeLogRecord
+)
+
+from openedx.core.djangoapps.content_libraries.api import (
+ library_component_usage_key, library_container_locator
+)
+
+from ..data import (
+ SourceContextKey, ModulestoreMigration, ModulestoreBlockMigrationResult,
+ ModulestoreBlockMigrationSuccess, ModulestoreBlockMigrationFailure
+)
+from .. import models
+
+
+__all__ = (
+ 'get_forwarding',
+ 'is_forwarded',
+ 'get_forwarding_for_blocks',
+ 'get_migrations',
+ 'get_migration_blocks',
+)
+
+
+def get_forwarding_for_blocks(source_keys: t.Iterable[UsageKey]) -> dict[UsageKey, ModulestoreBlockMigrationSuccess]:
+ """
+ Authoritatively determine how some Modulestore blocks have been migrated to Learning Core.
+
+ Returns a mapping from source usage keys to block migration data objects. Each block migration object
+ holds the target usage key and title. If a source key is missing from the mapping, then it has not
+ been authoritatively migrated.
+ """
+ sources = models.ModulestoreBlockSource.objects.filter(
+ key__in=[str(sk) for sk in source_keys]
+ ).select_related(
+ "forwarded__target__learning_package",
+ # For building component key
+ "forwarded__target__component__component_type",
+ # For building container key
+ "forwarded__target__container__section",
+ "forwarded__target__container__subsection",
+ "forwarded__target__container__unit",
+ # For determining title and version
+ "forwarded__change_log_record__new_version",
+ )
+ result = {}
+ for source in sources:
+ if source.forwarded and source.forwarded.target:
+ result[source.key] = _block_migration_success(
+ source_key=source.key,
+ target=source.forwarded.target,
+ change_log_record=source.forwarded.change_log_record,
+ )
+ return result
+
+
+def is_forwarded(source_key: SourceContextKey) -> bool:
+ """
+ Has this course or legacy library been authoratively migrated to Learning Core,
+ such that references to the source course/library should be forwarded to the target library?
+ """
+ return get_forwarding(source_key) is not None
+
+
+def get_forwarding(source_key: SourceContextKey) -> ModulestoreMigration | None:
+ """
+ Authoritatively determine how some Modulestore course or legacy library has been migrated to Learning Core.
+
+ If no such successful migration exists, returns None.
+
+ Note: This function may return None for a course or legacy lib that *has* been migrated 1+ times.
+ This just means that those migrations were non-forwarding. In user parlance, that is,
+ they have been "imported" but not truly "migrated".
+ """
+ try:
+ source = models.ModulestoreSource.objects.select_related(
+ # The following are used in _migration:
+ "forwarded__source",
+ "forwarded__target",
+ "forwarded__task_status",
+ "forwarded__target_collection",
+ ).get(
+ key=str(source_key)
+ )
+ except models.ModulestoreSource.DoesNotExist:
+ return None
+ if not source.forwarded:
+ return None
+ if source.forwarded.is_failed:
+ return None
+ return _migration(source.forwarded)
+
+
+def get_migrations(
+ source_key: SourceContextKey | None = None,
+ *,
+ target_key: LibraryLocatorV2 | None = None,
+ target_collection_slug: str | None = None,
+ task_uuid: UUID | None = None,
+ is_failed: bool | None = None,
+) -> t.Generator[ModulestoreMigration]:
+ """
+ Given some criteria, get all modulestore->LearningCore migrations.
+
+ Returns an iterable, ordered from NEWEST to OLDEST.
+
+ Please note: If you provide no filters, this will return an iterable across the whole
+ ModulestoreMigration table. Please paginate thoughtfully if you do that.
+ """
+ migrations = models.ModulestoreMigration.objects.all().select_related(
+ "source",
+ "target",
+ "target_collection",
+ "task_status",
+ )
+ if source_key:
+ migrations = migrations.filter(source__key=source_key)
+ if target_key:
+ migrations = migrations.filter(target__key=str(target_key))
+ if target_collection_slug:
+ migrations = migrations.filter(target_collection__key=target_collection_slug)
+ if task_uuid:
+ migrations = migrations.filter(task_status__uuid=str(task_uuid))
+ if is_failed is not None:
+ migrations = migrations.filter(is_failed=is_failed)
+ return (
+ _migration(migration)
+ for migration in migrations.order_by("-id") # primary key is a proxy for newness
+ )
+
+
+def get_migration_blocks(migration_pk: int) -> dict[UsageKey, ModulestoreBlockMigrationResult]:
+ """
+ Get details about the migrations of each individual block within a course/lib migration.
+ """
+ return {
+ block_migration.source.key: _block_migration_result(block_migration)
+ for block_migration in models.ModulestoreBlockMigration.objects.filter(
+ overall_migration_id=migration_pk
+ ).select_related(
+ "source",
+ "target__learning_package",
+ # For building component key
+ "target__component__component_type",
+ # For building container key.
+ # (Hard-coding these exact 3 container types here is not a good pattern, but it's what is needed
+ # here in order to avoid additional SELECTs while determining the container type).
+ "target__container__section",
+ "target__container__subsection",
+ "target__container__unit",
+ # For determining title and version
+ "change_log_record__new_version",
+ )
+ }
+
+
+def _migration(m: models.ModulestoreMigration) -> ModulestoreMigration:
+ """
+ Build a migration dataclass from the database row
+ """
+ return ModulestoreMigration(
+ pk=m.id,
+ source_key=m.source.key,
+ target_key=LibraryLocatorV2.from_string(m.target.key),
+ target_title=m.target.title,
+ target_collection_slug=(m.target_collection.key if m.target_collection else None),
+ target_collection_title=(m.target_collection.title if m.target_collection else None),
+ is_failed=m.is_failed,
+ task_uuid=m.task_status.uuid,
+ )
+
+
+def _block_migration_result(m: models.ModulestoreBlockMigration) -> ModulestoreBlockMigrationResult:
+ """
+ Build an instance of the migration result (successs/failure) dataclass from a database row
+ """
+ if m.target:
+ return _block_migration_success(
+ source_key=m.source.key,
+ target=m.target,
+ change_log_record=m.change_log_record,
+ )
+ return ModulestoreBlockMigrationFailure(
+ source_key=m.source.key,
+ unsupported_reason=(m.unsupported_reason or ""),
+ )
+
+
+def _block_migration_success(
+ source_key: UsageKey,
+ target: PublishableEntity,
+ change_log_record: DraftChangeLogRecord | None,
+) -> ModulestoreBlockMigrationSuccess:
+ """
+ Build an instance of the migration success dataclass
+ """
+ target_library_key = LibraryLocatorV2.from_string(target.learning_package.key)
+ target_key: LibraryUsageLocatorV2 | LibraryContainerLocator
+ if hasattr(target, "component"):
+ target_key = library_component_usage_key(target_library_key, target.component)
+ elif hasattr(target, "container"):
+ target_key = library_container_locator(target_library_key, target.container)
+ else:
+ raise ValueError(f"Entity is neither a container nor component: {target}")
+ # We expect that any successful BlockMigration (that is, one where `target is not None`)
+ # will also have a `change_log_record` with a non-None `new_version`. However, the data model
+ # does not guarantee that `change_log_record` nor `change_log_record.new_version` are non-
+ # None. So, just in case some bug in the modulestore_migrator or some manual modification of
+ # the database leads us to a situation where `target` is set but `change_log_record.new_version`
+ # is not, we have fallback behavior:
+ # * For target_title, use the latest draft's title, which is good enough, because the
+ # title is just there to help users.
+ # * For target_version_num, just use None, because we don't want downstream code to make decisions
+ # about syncing, etc based on incorrect version info.
+ target_version: PublishableEntityVersion | None = (
+ change_log_record.new_version if change_log_record else None
+ )
+ if target_version:
+ target_title = target_version.title
+ target_version_num = target_version.version_num
+ else:
+ latest_draft = get_draft_version(target)
+ target_title = latest_draft.title if latest_draft else ""
+ target_version_num = None
+ return ModulestoreBlockMigrationSuccess(
+ source_key=source_key,
+ target_entity_pk=target.id,
+ target_key=target_key,
+ target_title=target_title,
+ target_version_num=target_version_num,
+ )
diff --git a/cms/djangoapps/modulestore_migrator/api/write_api.py b/cms/djangoapps/modulestore_migrator/api/write_api.py
new file mode 100644
index 000000000000..4bef6c952767
--- /dev/null
+++ b/cms/djangoapps/modulestore_migrator/api/write_api.py
@@ -0,0 +1,94 @@
+"""
+API for kicking off new migrations
+"""
+from __future__ import annotations
+
+from celery.result import AsyncResult
+from opaque_keys.edx.locator import LibraryLocatorV2
+from openedx_learning.api.authoring import get_collection
+
+from openedx.core.types.user import AuthUser
+from openedx.core.djangoapps.content_libraries.api import get_library
+
+from ..data import SourceContextKey, CompositionLevel, RepeatHandlingStrategy
+from .. import tasks, models
+
+
+__all__ = (
+ 'start_migration_to_library',
+ 'start_bulk_migration_to_library'
+)
+
+
+def start_migration_to_library(
+ *,
+ user: AuthUser,
+ source_key: SourceContextKey,
+ target_library_key: LibraryLocatorV2,
+ target_collection_slug: str | None = None,
+ create_collection: bool = False,
+ composition_level: CompositionLevel,
+ repeat_handling_strategy: RepeatHandlingStrategy,
+ preserve_url_slugs: bool,
+ forward_source_to_target: bool | None
+) -> AsyncResult:
+ """
+ Import a course or legacy library into a V2 library (or, a collection within a V2 library).
+ """
+ return start_bulk_migration_to_library(
+ user=user,
+ source_key_list=[source_key],
+ target_library_key=target_library_key,
+ target_collection_slug_list=[target_collection_slug],
+ create_collections=create_collection,
+ composition_level=composition_level,
+ repeat_handling_strategy=repeat_handling_strategy,
+ preserve_url_slugs=preserve_url_slugs,
+ forward_source_to_target=forward_source_to_target,
+ )
+
+
+def start_bulk_migration_to_library(
+ *,
+ user: AuthUser,
+ source_key_list: list[SourceContextKey],
+ target_library_key: LibraryLocatorV2,
+ target_collection_slug_list: list[str | None] | None = None,
+ create_collections: bool = False,
+ composition_level: CompositionLevel,
+ repeat_handling_strategy: RepeatHandlingStrategy,
+ preserve_url_slugs: bool,
+ forward_source_to_target: bool | None,
+) -> AsyncResult:
+ """
+ Import a list of courses or legacy libraries into a V2 library (or, a collections within a V2 library).
+ """
+ target_library = get_library(target_library_key)
+ # get_library ensures that the library is connected to a learning package.
+ target_package_id: int = target_library.learning_package_id # type: ignore[assignment]
+
+ sources_pks: list[int] = []
+ for source_key in source_key_list:
+ source, _ = models.ModulestoreSource.objects.get_or_create(key=str(source_key))
+ sources_pks.append(source.id)
+
+ target_collection_pks: list[int | None] = []
+ if target_collection_slug_list:
+ for target_collection_slug in target_collection_slug_list:
+ if target_collection_slug:
+ target_collection_id = get_collection(target_package_id, target_collection_slug).id
+ target_collection_pks.append(target_collection_id)
+ else:
+ target_collection_pks.append(None)
+
+ return tasks.bulk_migrate_from_modulestore.delay(
+ user_id=user.id,
+ sources_pks=sources_pks,
+ target_library_key=str(target_library_key),
+ target_collection_pks=target_collection_pks,
+ create_collections=create_collections,
+ composition_level=composition_level.value,
+ repeat_handling_strategy=repeat_handling_strategy.value,
+ preserve_url_slugs=preserve_url_slugs,
+ forward_source_to_target=forward_source_to_target,
+ )
diff --git a/cms/djangoapps/modulestore_migrator/data.py b/cms/djangoapps/modulestore_migrator/data.py
index e42649557d67..529d4c78ad40 100644
--- a/cms/djangoapps/modulestore_migrator/data.py
+++ b/cms/djangoapps/modulestore_migrator/data.py
@@ -1,9 +1,23 @@
"""
Value objects
"""
+
from __future__ import annotations
+import typing as t
+from dataclasses import dataclass
from enum import Enum
+from uuid import UUID
+
+from django.utils.translation import gettext_lazy as _
+from opaque_keys.edx.keys import UsageKey
+from opaque_keys.edx.locator import (
+ CourseLocator,
+ LibraryContainerLocator,
+ LibraryLocator,
+ LibraryLocatorV2,
+ LibraryUsageLocatorV2,
+)
from openedx.core.djangoapps.content_libraries.api import ContainerType
@@ -70,3 +84,51 @@ def default(cls) -> RepeatHandlingStrategy:
Returns the default repeat handling strategy.
"""
return cls.Skip
+
+
+SourceContextKey: t.TypeAlias = CourseLocator | LibraryLocator
+
+
+@dataclass(frozen=True)
+class ModulestoreMigration:
+ """
+ Metadata on a migration of a course or legacy library to a v2 library in learning core.
+ """
+ pk: int
+ source_key: SourceContextKey
+ target_key: LibraryLocatorV2
+ target_title: str
+ target_collection_slug: str | None
+ target_collection_title: str | None
+ is_failed: bool
+ task_uuid: UUID # the UserTask which executed this migration
+
+
+@dataclass(frozen=True)
+class ModulestoreBlockMigrationResult:
+ """
+ Base class for a modulestore block that was part of an attempted migration to learning core.
+ """
+ source_key: UsageKey
+ is_failed: t.ClassVar[bool]
+
+
+@dataclass(frozen=True)
+class ModulestoreBlockMigrationSuccess(ModulestoreBlockMigrationResult):
+ """
+ Info on a modulestore block which has been successfully migrated into an LC entity
+ """
+ target_entity_pk: int
+ target_key: LibraryUsageLocatorV2 | LibraryContainerLocator
+ target_title: str
+ target_version_num: int | None
+ is_failed: t.ClassVar[bool] = False
+
+
+@dataclass(frozen=True)
+class ModulestoreBlockMigrationFailure(ModulestoreBlockMigrationResult):
+ """
+ Info on a modulestore block which failed to be migrated into LC
+ """
+ unsupported_reason: str
+ is_failed: t.ClassVar[bool] = True
diff --git a/cms/djangoapps/modulestore_migrator/migrations/0002_alter_modulestoremigration_task_status.py b/cms/djangoapps/modulestore_migrator/migrations/0002_alter_modulestoremigration_task_status.py
new file mode 100644
index 000000000000..ed3a299c30c0
--- /dev/null
+++ b/cms/djangoapps/modulestore_migrator/migrations/0002_alter_modulestoremigration_task_status.py
@@ -0,0 +1,20 @@
+# Generated by Django 4.2.24 on 2025-09-29 20:28
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('user_tasks', '0004_url_textfield'),
+ ('modulestore_migrator', '0001_initial'),
+ ]
+
+ operations = [
+ migrations.AlterField(
+ model_name='modulestoremigration',
+ name='task_status',
+ field=models.ForeignKey(help_text='Tracks the status of the task which is executing this migration. In a bulk migration, the same task can be multiple migrations', on_delete=django.db.models.deletion.RESTRICT, related_name='migrations', to='user_tasks.usertaskstatus'),
+ ),
+ ]
diff --git a/cms/djangoapps/modulestore_migrator/migrations/0003_modulestoremigration_is_failed.py b/cms/djangoapps/modulestore_migrator/migrations/0003_modulestoremigration_is_failed.py
new file mode 100644
index 000000000000..d7202023a94f
--- /dev/null
+++ b/cms/djangoapps/modulestore_migrator/migrations/0003_modulestoremigration_is_failed.py
@@ -0,0 +1,18 @@
+# Generated by Django 4.2.24 on 2025-10-21 23:37
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('modulestore_migrator', '0002_alter_modulestoremigration_task_status'),
+ ]
+
+ operations = [
+ migrations.AddField(
+ model_name='modulestoremigration',
+ name='is_failed',
+ field=models.BooleanField(default=False, help_text='is the migration failed?'),
+ ),
+ ]
diff --git a/cms/djangoapps/modulestore_migrator/migrations/0004_alter_modulestoreblockmigration_target_squashed_0005_modulestoreblockmigration_unsupported_reason.py b/cms/djangoapps/modulestore_migrator/migrations/0004_alter_modulestoreblockmigration_target_squashed_0005_modulestoreblockmigration_unsupported_reason.py
new file mode 100644
index 000000000000..f043e208dc35
--- /dev/null
+++ b/cms/djangoapps/modulestore_migrator/migrations/0004_alter_modulestoreblockmigration_target_squashed_0005_modulestoreblockmigration_unsupported_reason.py
@@ -0,0 +1,32 @@
+# Generated by Django 5.2.7 on 2025-11-26 06:35
+
+import django.db.models.deletion
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+ dependencies = [
+ ('modulestore_migrator', '0003_modulestoremigration_is_failed'),
+ ('oel_publishing', '0008_alter_draftchangelogrecord_options_and_more'),
+ ]
+
+ operations = [
+ migrations.AlterField(
+ model_name='modulestoreblockmigration',
+ name='target',
+ field=models.ForeignKey(
+ blank=True,
+ help_text='The target entity of this block migration, set to null if it fails to migrate',
+ null=True,
+ on_delete=django.db.models.deletion.CASCADE,
+ to='oel_publishing.publishableentity',
+ ),
+ ),
+ migrations.AddField(
+ model_name='modulestoreblockmigration',
+ name='unsupported_reason',
+ field=models.TextField(
+ blank=True, help_text='Reason if the block is unsupported and target is set to null', null=True
+ ),
+ ),
+ ]
diff --git a/cms/djangoapps/modulestore_migrator/migrations/0006_alter_modulestoreblocksource_forwarded_and_more.py b/cms/djangoapps/modulestore_migrator/migrations/0006_alter_modulestoreblocksource_forwarded_and_more.py
new file mode 100644
index 000000000000..a8ee5a0eb673
--- /dev/null
+++ b/cms/djangoapps/modulestore_migrator/migrations/0006_alter_modulestoreblocksource_forwarded_and_more.py
@@ -0,0 +1,30 @@
+# Generated by Django 5.2.9 on 2025-12-14 15:33
+
+import django.db.models.deletion
+import opaque_keys.edx.django.models
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('modulestore_migrator', '0004_alter_modulestoreblockmigration_target_squashed_0005_modulestoreblockmigration_unsupported_reason'),
+ ]
+
+ operations = [
+ migrations.AlterField(
+ model_name='modulestoreblocksource',
+ name='forwarded',
+ field=models.OneToOneField(help_text='If set, the system will forward references of this block source over to the target of this block migration', null=True, on_delete=django.db.models.deletion.SET_NULL, to='modulestore_migrator.modulestoreblockmigration'),
+ ),
+ migrations.AlterField(
+ model_name='modulestoreblocksource',
+ name='key',
+ field=opaque_keys.edx.django.models.UsageKeyField(help_text='Original usage key of the XBlock that has been imported.', max_length=255, unique=True),
+ ),
+ migrations.AlterField(
+ model_name='modulestoresource',
+ name='forwarded',
+ field=models.OneToOneField(blank=True, help_text='If set, the system will forward references of this source over to the target of this migration', null=True, on_delete=django.db.models.deletion.SET_NULL, to='modulestore_migrator.modulestoremigration'),
+ ),
+ ]
diff --git a/cms/djangoapps/modulestore_migrator/models.py b/cms/djangoapps/modulestore_migrator/models.py
index 05bea5bfe1e9..11f614c2ed03 100644
--- a/cms/djangoapps/modulestore_migrator/models.py
+++ b/cms/djangoapps/modulestore_migrator/models.py
@@ -6,16 +6,19 @@
from django.contrib.auth import get_user_model
from django.db import models
from django.utils.translation import gettext_lazy as _
-from user_tasks.models import UserTaskStatus
-
from model_utils.models import TimeStampedModel
from opaque_keys.edx.django.models import (
LearningContextKeyField,
UsageKeyField,
)
from openedx_learning.api.authoring_models import (
- LearningPackage, PublishableEntity, Collection, DraftChangeLog, DraftChangeLogRecord
+ Collection,
+ DraftChangeLog,
+ DraftChangeLogRecord,
+ LearningPackage,
+ PublishableEntity,
)
+from user_tasks.models import UserTaskStatus
from .data import CompositionLevel, RepeatHandlingStrategy
@@ -25,6 +28,23 @@
class ModulestoreSource(models.Model):
"""
A legacy learning context (course or library) which can be a source of a migration.
+
+ One source can be associated with multiple (successful or unsuccessful) ModulestoreMigrations.
+ If a source has been migrated multiple times, then at most one of them can be considered the
+ "official" or "authoritative" migration; this is indicated by setting the `forwarded` field to
+ that ModulestoreMigration object.
+
+ Note that `forwarded` can be NULL even when 1+ migrations have happened for this source. This just
+ means that none of them were authoritative. In other words, they were all "imports"/"copies" rather
+ than true "migrations".
+
+ In practice, as of Ulmo:
+ * The `forwarded` field is used to decide how to update legacy library_content references.
+ * When using the Libraries Migration UI in Studio, `forwarded` is always set to the first
+ successful ModulestoreMigration.
+ * When using the REST API directly, the default is to use the same behavior as the UI, but
+ clients can also explicitly specify the `forward_source_to_target` boolean param in order to
+ control whether `forwarded` is set to any given migration.
"""
key = LearningContextKeyField(
max_length=255,
@@ -37,11 +57,10 @@ class ModulestoreSource(models.Model):
blank=True,
on_delete=models.SET_NULL,
help_text=_('If set, the system will forward references of this source over to the target of this migration'),
- related_name="forwards",
)
def __str__(self):
- return f"{self.__class__.__name__}('{self.key}')"
+ return f"{self.key}"
__repr__ = __str__
@@ -107,10 +126,14 @@ class ModulestoreMigration(models.Model):
)
## MIGRATION ARTIFACTS
- task_status = models.OneToOneField(
+ task_status = models.ForeignKey(
UserTaskStatus,
on_delete=models.RESTRICT,
- help_text=_("Tracks the status of the task which is executing this migration"),
+ help_text=_(
+ "Tracks the status of the task which is executing this migration. "
+ "In a bulk migration, the same task can be multiple migrations"
+ ),
+ related_name="migrations",
)
change_log = models.ForeignKey(
DraftChangeLog,
@@ -127,6 +150,17 @@ class ModulestoreMigration(models.Model):
"We temporarily save the staged content to allow for troubleshooting of failed migrations."
)
)
+ # Mostly used in bulk migrations. The `UserTaskStatus` represents the status of the entire bulk migration;
+ # a `FAILED` status means that the entire bulk-migration has failed.
+ # Each `ModulestoreMigration` saves the data of the migration of each legacy library.
+ # The `is_failed` value is to keep track a failed legacy library in the bulk migration,
+ # but allow continuing with the migration of the rest of the legacy libraries.
+ is_failed = models.BooleanField(
+ default=False,
+ help_text=_(
+ "is the migration failed?"
+ ),
+ )
def __str__(self):
return (
@@ -145,6 +179,9 @@ def __repr__(self):
class ModulestoreBlockSource(TimeStampedModel):
"""
A legacy block usage (in a course or library) which can be a source of a block migration.
+
+ The semantics of `forwarded` directly mirror those of `ModulestoreSource.forwarded`. Please see
+ that class's docstring for details.
"""
overall_source = models.ForeignKey(
ModulestoreSource,
@@ -153,6 +190,7 @@ class ModulestoreBlockSource(TimeStampedModel):
)
key = UsageKeyField(
max_length=255,
+ unique=True,
help_text=_('Original usage key of the XBlock that has been imported.'),
)
forwarded = models.OneToOneField(
@@ -160,11 +198,10 @@ class ModulestoreBlockSource(TimeStampedModel):
null=True,
on_delete=models.SET_NULL,
help_text=_(
- 'If set, the system will forward references of this block source over to the target of this block migration'
+ 'If set, the system will forward references of this block source over to the '
+ 'target of this block migration'
),
- related_name="forwards",
)
- unique_together = [("overall_source", "key")]
def __str__(self):
return f"{self.__class__.__name__}('{self.key}')"
@@ -195,6 +232,9 @@ class ModulestoreBlockMigration(TimeStampedModel):
target = models.ForeignKey(
PublishableEntity,
on_delete=models.CASCADE,
+ help_text=_('The target entity of this block migration, set to null if it fails to migrate'),
+ null=True,
+ blank=True,
)
change_log_record = models.OneToOneField(
DraftChangeLogRecord,
@@ -203,10 +243,16 @@ class ModulestoreBlockMigration(TimeStampedModel):
null=True,
on_delete=models.SET_NULL,
)
+ unsupported_reason = models.TextField(
+ null=True,
+ blank=True,
+ help_text=_('Reason if the block is unsupported and target is set to null'),
+ )
class Meta:
unique_together = [
('overall_migration', 'source'),
+ # By default defining a unique index on a nullable column will only enforce unicity of non-null values.
('overall_migration', 'target'),
]
diff --git a/cms/djangoapps/modulestore_migrator/rest_api/v1/serializers.py b/cms/djangoapps/modulestore_migrator/rest_api/v1/serializers.py
index df981e896176..643d94d2250c 100644
--- a/cms/djangoapps/modulestore_migrator/rest_api/v1/serializers.py
+++ b/cms/djangoapps/modulestore_migrator/rest_api/v1/serializers.py
@@ -5,16 +5,30 @@
from opaque_keys import InvalidKeyError
from opaque_keys.edx.keys import LearningContextKey
from opaque_keys.edx.locator import LibraryLocatorV2
+from openedx_learning.api.authoring_models import Collection
from rest_framework import serializers
+from user_tasks.models import UserTaskStatus
from user_tasks.serializers import StatusSerializer
from cms.djangoapps.modulestore_migrator.data import CompositionLevel, RepeatHandlingStrategy
-from cms.djangoapps.modulestore_migrator.models import ModulestoreMigration
+from cms.djangoapps.modulestore_migrator.models import (
+ ModulestoreMigration,
+ ModulestoreSource,
+)
-class ModulestoreMigrationSerializer(serializers.ModelSerializer):
+class LibraryMigrationCollectionSerializer(serializers.ModelSerializer):
"""
- Serializer for the course to library import creation API.
+ Serializer for the target collection of a library migration.
+ """
+ class Meta:
+ model = Collection
+ fields = ["key", "title"]
+
+
+class ModulestoreMigrationSerializer(serializers.Serializer):
+ """
+ Serializer for the course or legacylibrary to library V2 import creation API.
"""
source = serializers.CharField( # type: ignore[assignment]
@@ -22,7 +36,7 @@ class ModulestoreMigrationSerializer(serializers.ModelSerializer):
required=True,
)
target = serializers.CharField(
- help_text="The target library key to import into.",
+ help_text="The target content library V2 key to import into.",
required=True,
)
composition_level = serializers.ChoiceField(
@@ -40,7 +54,7 @@ class ModulestoreMigrationSerializer(serializers.ModelSerializer):
preserve_url_slugs = serializers.BooleanField(
help_text="If true, current slugs will be preserved.",
required=False,
- default=True,
+ default=False,
)
target_collection_slug = serializers.CharField(
help_text="The target collection slug within the library to import into. Optional.",
@@ -48,24 +62,27 @@ class ModulestoreMigrationSerializer(serializers.ModelSerializer):
allow_blank=True,
default=None,
)
+ create_collection = serializers.BooleanField(
+ help_text=(
+ "If true and `target_collection_slug` is not set, "
+ "create the collections in the library where the import will be made"
+ ),
+ required=False,
+ default=False,
+ )
+ target_collection = LibraryMigrationCollectionSerializer(required=False)
forward_source_to_target = serializers.BooleanField(
help_text="Forward references of this block source over to the target of this block migration.",
required=False,
+ allow_null=True,
+ default=None, # Note: "None" means "unspecified"
+ )
+ is_failed = serializers.BooleanField(
+ help_text="It is true if this migration is failed",
+ required=False,
default=False,
)
- class Meta:
- model = ModulestoreMigration
- fields = [
- 'source',
- 'target',
- 'target_collection_slug',
- 'composition_level',
- 'repeat_handling_strategy',
- 'preserve_url_slugs',
- 'forward_source_to_target',
- ]
-
def get_fields(self):
fields = super().get_fields()
request = self.context.get('request')
@@ -100,19 +117,74 @@ def get_forward_source_to_target(self, obj: ModulestoreMigration):
def to_representation(self, instance):
"""
- Override to customize the serialized representation."""
+ Override to customize the serialized representation.
+ """
data = super().to_representation(instance)
# Custom logic for forward_source_to_target during serialization
data['forward_source_to_target'] = self.get_forward_source_to_target(instance)
return data
-class StatusWithModulestoreMigrationSerializer(StatusSerializer):
+class BulkModulestoreMigrationSerializer(ModulestoreMigrationSerializer):
+ """
+ Serializer for a bulk migration (of several courses or legacy libraries) to a V2 library.
+ """
+ sources = serializers.ListField(
+ child=serializers.CharField(),
+ help_text="The list of sources course or legacy library keys to import from.",
+ required=True,
+ )
+
+ target_collection_slug_list = serializers.ListField(
+ child=serializers.CharField(),
+ help_text="The list of target collection slugs within the library to import into. Optional.",
+ required=False,
+ allow_empty=True,
+ default=None,
+ )
+
+ create_collections = serializers.BooleanField(
+ help_text=(
+ "If true and `target_collection_slug_list` is not set, "
+ "create the collections in the library where the import will be made"
+ ),
+ required=False,
+ default=False,
+ )
+
+ def get_fields(self):
+ fields = super().get_fields()
+ fields.pop("source", None)
+ fields.pop("target_collection_slug", None)
+ return fields
+
+ def validate_sources(self, value):
+ """
+ Validate all the source key format
+ """
+ validated_sources = []
+ for v in value:
+ try:
+ validated_sources.append(LearningContextKey.from_string(v))
+ except InvalidKeyError as exc:
+ raise serializers.ValidationError(f"Invalid source key: {str(exc)}") from exc
+ return validated_sources
+
+ def to_representation(self, instance):
+ """
+ Override to customize the serialized representation.
+ """
+ if isinstance(instance, list):
+ return [super().to_representation(obj) for obj in instance]
+ return super().to_representation(instance)
+
+
+class StatusWithModulestoreMigrationsSerializer(StatusSerializer):
"""
- Serializer for the import task status.
+ Serializer for the import task status, including 1+ migration objects.
"""
- parameters = ModulestoreMigrationSerializer(source='modulestoremigration')
+ parameters = ModulestoreMigrationSerializer(source='migrations', many=True)
class Meta:
model = StatusSerializer.Meta.model
@@ -125,3 +197,95 @@ def get_fields(self):
fields = super().get_fields()
fields.pop('name', None)
return fields
+
+
+class MigrationInfoSerializer(serializers.Serializer):
+ """
+ Serializer for the migration info
+ """
+
+ source_key = serializers.CharField()
+ target_key = serializers.CharField()
+ target_title = serializers.CharField()
+ target_collection_key = serializers.CharField(
+ source="target_collection_slug",
+ allow_null=True
+ )
+ target_collection_title = serializers.CharField(
+ allow_null=True
+ )
+
+
+class MigrationInfoResponseSerializer(serializers.Serializer):
+ """
+ Serializer for the migrations info view response
+ """
+ def to_representation(self, instance):
+ return {
+ str(key): MigrationInfoSerializer(value, many=True).data
+ for key, value in instance.items()
+ }
+
+
+class LibraryMigrationCourseSourceSerializer(serializers.ModelSerializer):
+ """
+ Serializer for the source course of a library migration.
+ """
+ display_name = serializers.SerializerMethodField()
+
+ class Meta:
+ model = ModulestoreSource
+ fields = ['key', 'display_name']
+
+ def get_display_name(self, obj):
+ """
+ Return the display name of the source course
+ """
+ return self.context["course_names"].get(str(obj.key), None)
+
+
+class LibraryMigrationCourseSerializer(serializers.ModelSerializer):
+ """
+ Serializer for the course or legacylibrary migrations to V2 library.
+ """
+ task_uuid = serializers.UUIDField(source='task_status.uuid', read_only=True)
+ source = LibraryMigrationCourseSourceSerializer() # type: ignore[assignment]
+ target_collection = LibraryMigrationCollectionSerializer(required=False)
+ state = serializers.SerializerMethodField()
+ progress = serializers.SerializerMethodField()
+
+ class Meta:
+ model = ModulestoreMigration
+ fields = [
+ 'task_uuid',
+ 'source',
+ 'target_collection',
+ 'state',
+ 'progress',
+ ]
+
+ def get_state(self, obj: ModulestoreMigration):
+ """
+ Return the state of the migration.
+ """
+ if obj.is_failed or obj.task_status.state in [UserTaskStatus.FAILED, UserTaskStatus.CANCELED]:
+ return UserTaskStatus.FAILED
+ elif obj.task_status.state == UserTaskStatus.SUCCEEDED:
+ return UserTaskStatus.SUCCEEDED
+
+ return UserTaskStatus.IN_PROGRESS
+
+ def get_progress(self, obj: ModulestoreMigration):
+ """
+ Return the progress of the migration.
+ """
+ return obj.task_status.completed_steps / obj.task_status.total_steps
+
+
+class BlockMigrationInfoSerializer(serializers.Serializer):
+ """
+ Serializer for the block migration info.
+ """
+ source_key = serializers.CharField()
+ target_key = serializers.CharField(allow_null=True)
+ unsupported_reason = serializers.CharField(allow_null=True)
diff --git a/cms/djangoapps/modulestore_migrator/rest_api/v1/urls.py b/cms/djangoapps/modulestore_migrator/rest_api/v1/urls.py
index 22d509f16910..4d2c92b6af04 100644
--- a/cms/djangoapps/modulestore_migrator/rest_api/v1/urls.py
+++ b/cms/djangoapps/modulestore_migrator/rest_api/v1/urls.py
@@ -1,11 +1,15 @@
"""
Course to Library Import API v1 URLs.
"""
-
from rest_framework.routers import SimpleRouter
-from .views import MigrationViewSet
+
+from .views import (
+ BulkMigrationViewSet,
+ MigrationViewSet,
+)
ROUTER = SimpleRouter()
-ROUTER.register(r'migrations', MigrationViewSet)
+ROUTER.register(r'migrations', MigrationViewSet, basename='migrations')
+ROUTER.register(r'bulk_migration', BulkMigrationViewSet, basename='bulk-migration')
urlpatterns = ROUTER.urls
diff --git a/cms/djangoapps/modulestore_migrator/rest_api/v1/views.py b/cms/djangoapps/modulestore_migrator/rest_api/v1/views.py
index feeafb1b40d7..f84f87a23c85 100644
--- a/cms/djangoapps/modulestore_migrator/rest_api/v1/views.py
+++ b/cms/djangoapps/modulestore_migrator/rest_api/v1/views.py
@@ -3,135 +3,371 @@
"""
import logging
+import edx_api_doc_tools as apidocs
from edx_rest_framework_extensions.auth.jwt.authentication import JwtAuthentication
from edx_rest_framework_extensions.auth.session.authentication import SessionAuthenticationAllowInactiveUser
-from rest_framework.permissions import IsAdminUser
-from rest_framework.response import Response
from rest_framework import status
+from rest_framework.decorators import action
+from rest_framework.exceptions import PermissionDenied
+from rest_framework.response import Response
from user_tasks.models import UserTaskStatus
from user_tasks.views import StatusViewSet
-from cms.djangoapps.modulestore_migrator.api import start_migration_to_library
+from cms.djangoapps.modulestore_migrator import api as migrator_api
+from common.djangoapps.student import auth
+from openedx.core.djangoapps.content_libraries import api as lib_api
from openedx.core.lib.api.authentication import BearerAuthenticationAllowInactiveUser
-from .serializers import ModulestoreMigrationSerializer, StatusWithModulestoreMigrationSerializer
-
+from ...data import (
+ CompositionLevel, RepeatHandlingStrategy,
+)
+from .serializers import (
+ BulkModulestoreMigrationSerializer,
+ ModulestoreMigrationSerializer,
+ StatusWithModulestoreMigrationsSerializer,
+)
log = logging.getLogger(__name__)
+_error_responses = {
+ 400: "Request malformed.",
+ 401: "Requester is not authenticated.",
+ 403: "Permission denied",
+}
+
+
+@apidocs.schema_for(
+ "list",
+ """
+ List all migration and bulk-migration tasks started by the current user.
+
+ The response is a paginated series of migration task status objects, ordered
+ by the time at which the migration was started, newest first.
+ See `POST /api/modulestore_migrator/v1/migrations` for details of each object's schema.
+ """,
+)
+@apidocs.schema_for(
+ "retrieve",
+ """
+ Get the status of particular migration or bulk-migration task by its UUID.
+
+ The response is a migration task status object.
+ See `POST /api/modulestore_migrator/v1/migrations` for details on its schema.
+ """,
+)
class MigrationViewSet(StatusViewSet):
"""
- Import course content from modulestore into a content library.
-
- This viewset handles the import process, including creating the import task and
- retrieving the status of the import task. Meant to be used by admin users only.
-
- API Endpoints
- ------------
- POST /api/modulestore_migrator/v1/migrations/
- Start the import process.
-
- Request body:
- {
- "source": "",
- "target": "",
- "composition_level": "", # Optional, defaults to "component"
- "target_collection_slug": "", # Optional
- "repeat_handling_strategy": "" # Optional, defaults to Skip
- "preserve_url_slugs": "" # Optional, defaults to true
- }
+ JSON HTTP API to create and check on ModuleStore-to-Learning-Core migration tasks.
+ """
+
+ authentication_classes = (
+ BearerAuthenticationAllowInactiveUser,
+ JwtAuthentication,
+ SessionAuthenticationAllowInactiveUser,
+ )
+ serializer_class = StatusWithModulestoreMigrationsSerializer
+
+ # DELETE is not allowed, as we want to preserve all task status objects.
+ # Instead, users can POST to /cancel to cancel running tasks.
+ http_method_names = ["get", "post"]
+
+ lookup_field = "uuid"
+
+ def get_queryset(self):
+ """
+ Override the default queryset to filter by the migration event and user.
+ """
+ return StatusViewSet.queryset.filter(
+ migrations__isnull=False,
+ # The filter for `user` here is essentially the auth strategy for the /list and /retreive
+ # endpoints. Basically: you can view migrations if and only if you started them.
+ # Future devs: If you ever refactor this view to remove the user filter, be sure to enforce
+ # permissions some other way.
+ user=self.request.user
+ ).distinct().order_by("-created")
+
+ @apidocs.schema()
+ @action(detail=True, methods=['post'])
+ def cancel(self, request, *args, **kwargs):
+ """
+ Cancel a particular migration or bulk-migration task.
+
+ The response is a migration task status object.
+ See `POST /api/modulestore_migrator/v1/migrations` for details on its schema.
+
+ This endpoint is currently reserved for site-wide administrators.
+ """
+ # TODO: This should check some sort of "allowed to cancel/migrations" permission
+ # rather than directly looking at the GlobalStaff role.
+ # https://github.com/openedx/edx-platform/issues/37791
+ if not request.user.is_staff:
+ raise PermissionDenied("Only site administrators can cancel migration tasks.")
+ return super().cancel(request, *args, **kwargs)
+
+ @apidocs.schema(
+ body=ModulestoreMigrationSerializer,
+ responses={
+ 201: StatusWithModulestoreMigrationsSerializer,
+ **_error_responses,
+ },
+ )
+ def create(self, request, *args, **kwargs):
+ """
+ Begin a transfer of content from course or legacy library into a content library.
+
+ Required parameters:
+ * A **source** key, which identifies the course or legacy library containing the items be migrated.
+ * A **target** key, which identifies the content library to which items will be migrated.
+
+ Optional parameters:
+
+ * The **target_collection_slug**, which identifies an *existing* collection within the target that
+ should hold the migrated items. If not specified, items will be added to the target library without
+ any collection, unless:
+ * If this source was previously migrated to a collection and the **repeat_handling_strategy** (described
+ below) is not set to *fork*, then that same collection will be re-used.
+ * If **create_collection** is specified as *true*, then the items will be added to a new collection, with
+ a name and slug based on the source's title, but not conflicting with any existing collection.
+ * The **composition_level** (*component*, *unit*, *subsection*, *section*) indicates the highest level of
+ hierarchy to be transferred. Default is *component*. To maximally preserve the source structure,
+ specify *section*.
+ * The **repeat_handling_strategy** specifies how the system should handle source items which have
+ previously been migrated to the target.
+ * Specify *skip* to prefer the existing target item. This is the default.
+ * Specify *update* to update the existing target item with the latest source content.
+ * Specify *fork* to create a new target item with the source content.
+ * Specify **preserve_url_slugs** as *true* in order to use the source-provided block IDs
+ (a.k.a. "URL slugs", "url_names"). Otherwise, the system will use each source item's title
+ to auto-generate an ID in the target context.
+ * Specify **forward_source_to_target** as *true* in order to establish a mapping from the source items to the
+ target items (specifically, the mapping is stored by the *forwarded* field of the
+ *modulestore_migrator_modulestoresource* and *modulestore_migrator_modulestoreblocksource* database tables).
+ * **Example**: Specify *true* if you are permanently migrating legacy library content into a content
+ library, and want course references to the legacy library content to automatically be mapped to new
+ content library.
+ * **Example**: Specify *false* if you are migrating legacy library content into a content
+ library, but do *not* want course references to the legacy library content to be mapped to new
+ content library.
+ * **Example**: Specify *false* if you are copying course content into a content library, but do not
+ want to persist a link between the source source content and destination library contenet.
+ * **Defaults** to *false* if the source has already been mapped to a target by a successful migration,
+ and defaults to *true* if not. In other words, by default, establish the mapping only if it wouldn't
+ override an existing mapping.
Example request:
- {
- "source": "course-v1:edX+DemoX+2014_T1",
- "target": "library-v1:org1+lib_1",
- "composition_level": "unit",
- "repeat_handling_strategy": "update",
- "preserve_url_slugs": true
- }
+ ```json
+ {
+ "source": "course-v1:MyOrganization+MyCourse+MyRun",
+ "target": "lib-collection:MyOrganization:MyUlmoLibrary",
+ "target_collection_slug": "MyCollection",
+ "create_collection": true,
+ "composition_level": "unit",
+ "repeat_handling_strategy": "update",
+ "preserve_url_slugs": true
+ }
+ ```
+
+ The migration task will take anywhere from seconds to minutes, depending on the size of the source
+ content. This API's response will tell the initial status of the migration task, including:
+
+ * The **state**, whose values include:
+ * _Pending_: The migration task is waiting to be picked up by a Studio worker process.
+ * _Succeeded_: The migration task finished without fatal errors.
+ * _Failed_: A fatal error occured during the migration task.
+ * _Canceled_: An administrator canceled the migration task.
+ * Any other **state** value indicates the migration is actively in progress.
+ * The **state_text**, the localized version of **state**.
+ * The **artifacts**, a list of URLs pointing to additional diagnostic information created
+ during that task. In the current version of this API, successful migrations will have zero
+ artifacts and failed migrations will have one artifact.
+ * The **uuid**, which be used to retrieve an updated status on this migration later, via
+ *GET /api/modulestore_migrator/v1/migrations/$uuid*
+ * The **parameters**, a singleton list whose elemenet indicates the parameters which were
+ used to start this migration task.
Example response:
- {
- "state": "Succeeded",
- "state_text": "Succeeded", # Translation into the current language of the current state
- "completed_steps": 11,
- "total_steps": 11,
- "attempts": 1,
- "created": "2025-05-14T22:24:37.048539Z",
- "modified": "2025-05-14T22:24:59.128068Z",
- "artifacts": [],
- "uuid": "3de23e5d-fd34-4a6f-bf02-b183374120f0",
- "parameters": {
- "source": "course-v1:OpenedX+DemoX+DemoCourse",
+ ```json
+ {
+ "state": "Parsing staged OLX",
+ "state_text": "Parsing staged OLX",
+ "completed_steps": 4,
+ "total_steps": 12,
+ "attempts": 1,
+ "created": "2025-05-14T22:24:37.048539Z",
+ "modified": "2025-05-14T22:24:59.128068Z",
+ "artifacts": [],
+ "uuid": "3de23e5d-fd34-4a6f-bf02-b183374120f0",
+ "parameters": [
+ {
+ "source": "course-v1:MyOrganization+MyCourse+MyRun",
+ "target": "lib:MyOrganization:MyUlmoLibrary",
"composition_level": "unit",
"repeat_handling_strategy": "update",
"preserve_url_slugs": true
}
- }
+ ]
+ }
+ ```
- GET /api/modulestore_migrator/v1/migrations//
- Get the status of the import task.
+ This API requires that the requester have author access on both the source and target.
+ """
+ serializer_data = ModulestoreMigrationSerializer(data=request.data)
+ serializer_data.is_valid(raise_exception=True)
+ validated_data = serializer_data.validated_data
+ if not auth.has_studio_write_access(request.user, validated_data['source']):
+ raise PermissionDenied("Requester is not an author on the source.")
+ lib_api.require_permission_for_library_key(
+ validated_data['target'],
+ request.user,
+ lib_api.permissions.CAN_EDIT_THIS_CONTENT_LIBRARY,
+ )
+ task = migrator_api.start_migration_to_library(
+ user=request.user,
+ source_key=validated_data['source'],
+ target_library_key=validated_data['target'],
+ target_collection_slug=validated_data['target_collection_slug'],
+ composition_level=CompositionLevel(validated_data['composition_level']),
+ create_collection=validated_data['create_collection'],
+ repeat_handling_strategy=RepeatHandlingStrategy(validated_data['repeat_handling_strategy']),
+ preserve_url_slugs=validated_data['preserve_url_slugs'],
+ forward_source_to_target=validated_data['forward_source_to_target'],
+ )
+ task_status = UserTaskStatus.objects.get(task_id=task.id)
+ serializer = self.get_serializer(task_status)
+ return Response(serializer.data, status=status.HTTP_201_CREATED)
- Example response:
- {
- "state": "Importing staged content structure",
- "state_text": "Importing staged content structure",
- "completed_steps": 6,
- "total_steps": 11,
- "attempts": 1,
- "created": "2025-05-14T22:24:37.048539Z",
- "modified": "2025-05-14T22:24:59.128068Z",
- "artifacts": [],
- "uuid": "3de23e5d-fd34-4a6f-bf02-b183374120f0",
- "parameters": {
- "source": "course-v1:OpenedX+DemoX+DemoCourse2",
- "composition_level": "component",
- "repeat_handling_strategy": "skip",
- "preserve_url_slugs": false
- }
- }
+
+class BulkMigrationViewSet(StatusViewSet):
+ """
+ JSON HTTP API to bulk-create ModuleStore-to-Learning-Core migration tasks.
"""
- permission_classes = (IsAdminUser,)
authentication_classes = (
BearerAuthenticationAllowInactiveUser,
JwtAuthentication,
SessionAuthenticationAllowInactiveUser,
)
- serializer_class = StatusWithModulestoreMigrationSerializer
+ serializer_class = StatusWithModulestoreMigrationsSerializer
- def get_queryset(self):
- """
- Override the default queryset to filter by the import event and user.
- """
- return StatusViewSet.queryset.filter(modulestoremigration__isnull=False, user=self.request.user)
+ # Because bulk-migration tasks are just special migration tasks, they are listed as part of:
+ # GET .../migrations/
+ # To avoid having 2 APIs that do (mostly) the same thing, we nix this endpoint:
+ # GET .../bulk_migration//cancel
+ # which we inherited from StatusViewSet.
+ # Furthermore, DELETE is not allowed, as we want to preserve all task status objects.
+ # That just leaves us with POST.
+ http_method_names = ["post"]
+ @apidocs.schema(
+ body=BulkModulestoreMigrationSerializer,
+ responses={
+ 201: StatusWithModulestoreMigrationsSerializer,
+ **_error_responses,
+ },
+ )
def create(self, request, *args, **kwargs):
"""
- Handle the import task creation.
- """
+ Transfer content from multiple courses or legacy libraries into a content library.
- serializer_data = ModulestoreMigrationSerializer(data=request.data)
+ Create a migration task to import multiple courses or legacy libraries into a single content library.
+ This is bulk version of `POST /api/modulestore_migrator/v1/migrations`. See that endpoint's documentation
+ for details on the meanings of the request and response values.
+
+ **Request body**:
+ ```json
+ {
+ "sources": ["", ""],
+ "target": "",
+ "composition_level": "", # Optional, defaults to "component"
+ "target_collection_slugs": ["", ""], # Optional
+ "create_collections": "" # Optional, defaults to false
+ "repeat_handling_strategy": "" # Optional, defaults to Skip
+ "preserve_url_slugs": "" # Optional, defaults to true
+ }
+ ```
+
+ **Example request:**
+ ```json
+ {
+ "sources": ["course-v1:edX+DemoX+2014_T1", "course-v1:edX+DemoX+2014_T2"],
+ "target": "library-v1:org1+lib_1",
+ "composition_level": "unit",
+ "repeat_handling_strategy": "update",
+ "preserve_url_slugs": true,
+ "create_collections": true
+ }
+ ```
+
+ **Example response:**
+ ```json
+ {
+ "state": "Succeeded",
+ "state_text": "Succeeded", # Translation into the current language of the current state
+ "completed_steps": 11,
+ "total_steps": 11,
+ "attempts": 1,
+ "created": "2025-05-14T22:24:37.048539Z",
+ "modified": "2025-05-14T22:24:59.128068Z",
+ "artifacts": [],
+ "uuid": "3de23e5d-fd34-4a6f-bf02-b183374120f0",
+ "parameters": [
+ {
+ "source": "course-v1:edX+DemoX+2014_T1",
+ "composition_level": "unit",
+ "repeat_handling_strategy": "update",
+ "preserve_url_slugs": true
+ },
+ {
+ "source": "course-v1:edX+DemoX+2014_T2",
+ "composition_level": "unit",
+ "repeat_handling_strategy": "update",
+ "preserve_url_slugs": true
+ },
+ ]
+ }
+ ```
+
+ This API requires that the requester have author access on both the source and target.
+ """
+ serializer_data = BulkModulestoreMigrationSerializer(data=request.data)
serializer_data.is_valid(raise_exception=True)
validated_data = serializer_data.validated_data
-
- try:
- task = start_migration_to_library(
- user=request.user,
- source_key=validated_data['source'],
- target_library_key=validated_data['target'],
- target_collection_slug=validated_data['target_collection_slug'],
- composition_level=validated_data['composition_level'],
- repeat_handling_strategy=validated_data['repeat_handling_strategy'],
- preserve_url_slugs=validated_data['preserve_url_slugs'],
- forward_source_to_target=validated_data['forward_source_to_target'],
- )
- except NotImplementedError as e:
- log.exception(str(e))
- return Response({'error': str(e)}, status=status.HTTP_400_BAD_REQUEST)
+ for source_key in validated_data['sources']:
+ if not auth.has_studio_write_access(request.user, source_key):
+ raise PermissionDenied(
+ f"Requester is not an author on the source: {source_key}. No migrations performed."
+ )
+ lib_api.require_permission_for_library_key(
+ validated_data['target'],
+ request.user,
+ lib_api.permissions.CAN_EDIT_THIS_CONTENT_LIBRARY,
+ )
+ task = migrator_api.start_bulk_migration_to_library(
+ user=request.user,
+ source_key_list=validated_data['sources'],
+ target_library_key=validated_data['target'],
+ target_collection_slug_list=validated_data['target_collection_slug_list'],
+ create_collections=validated_data['create_collections'],
+ composition_level=CompositionLevel(validated_data['composition_level']),
+ repeat_handling_strategy=RepeatHandlingStrategy(validated_data['repeat_handling_strategy']),
+ preserve_url_slugs=validated_data['preserve_url_slugs'],
+ forward_source_to_target=validated_data['forward_source_to_target'],
+ )
task_status = UserTaskStatus.objects.get(task_id=task.id)
serializer = self.get_serializer(task_status)
return Response(serializer.data, status=status.HTTP_201_CREATED)
+
+ def cancel(self, request, *args, **kwargs):
+ """
+ Remove the `POST .../cancel` endpoint, which was inherited from StatusViewSet.
+
+ Bulk-migration tasks and migration tasks, under the hood, are the same thing.
+ So, bulk-migration tasks can be cancelled with:
+ POST .../migrations//cancel
+
+ We disable this endpoint to avoid confusion.
+ """
+ raise NotImplementedError
diff --git a/cms/djangoapps/modulestore_migrator/tasks.py b/cms/djangoapps/modulestore_migrator/tasks.py
index 0c8520988738..e0b0b0fe2ed2 100644
--- a/cms/djangoapps/modulestore_migrator/tasks.py
+++ b/cms/djangoapps/modulestore_migrator/tasks.py
@@ -9,23 +9,26 @@
from dataclasses import dataclass
from datetime import datetime, timezone
from enum import Enum
-from itertools import groupby
+from gettext import ngettext
from celery import shared_task
from celery.utils.log import get_task_logger
from django.core.exceptions import ObjectDoesNotExist
+from django.db import transaction
from django.utils.text import slugify
+from django.utils.translation import gettext_lazy as _
from edx_django_utils.monitoring import set_code_owner_attribute_from_module
from lxml import etree
from lxml.etree import _ElementTree as XmlTree
from opaque_keys import InvalidKeyError
-from opaque_keys.edx.keys import CourseKey, UsageKey
+from opaque_keys.edx.keys import UsageKey
from opaque_keys.edx.locator import (
+ BlockUsageLocator,
CourseLocator,
LibraryContainerLocator,
LibraryLocator,
LibraryLocatorV2,
- LibraryUsageLocatorV2
+ LibraryUsageLocatorV2,
)
from openedx_learning.api import authoring as authoring_api
from openedx_learning.api.authoring_models import (
@@ -34,20 +37,24 @@
ComponentType,
LearningPackage,
PublishableEntity,
- PublishableEntityVersion
+ PublishableEntityVersion,
)
from user_tasks.tasks import UserTask, UserTaskStatus
+from xblock.core import XBlock
+from xblock.plugin import PluginMissingError
from common.djangoapps.split_modulestore_django.models import SplitModulestoreCourseIndex
+from common.djangoapps.util.date_utils import DEFAULT_DATE_TIME_FORMAT, strftime_localized
from openedx.core.djangoapps.content_libraries import api as libraries_api
from openedx.core.djangoapps.content_libraries.api import ContainerType, get_library
from openedx.core.djangoapps.content_staging import api as staging_api
from xmodule.modulestore import exceptions as modulestore_exceptions
from xmodule.modulestore.django import modulestore
+from . import models, data
from .constants import CONTENT_STAGING_PURPOSE_TEMPLATE
-from .data import CompositionLevel, RepeatHandlingStrategy
-from .models import ModulestoreBlockMigration, ModulestoreBlockSource, ModulestoreMigration, ModulestoreSource
+from .data import CompositionLevel, RepeatHandlingStrategy, SourceContextKey
+from .api.read_api import get_migrations, get_migration_blocks
log = get_task_logger(__name__)
@@ -71,19 +78,35 @@ class MigrationStep(Enum):
MAPPING_OLD_TO_NEW = 'Saving map of legacy content to migrated content'
FORWARDING = 'Forwarding legacy content to migrated content'
POPULATING_COLLECTION = 'Assigning imported items to the specified collection'
+ BULK_MIGRATION_PREFIX = 'Migrating legacy content'
-class _MigrationTask(UserTask):
+class _BulkMigrationTask(UserTask):
"""
- Base class for migrate_to_modulestore
+ Base class for bulk_migrate_from_modulestore
"""
@staticmethod
def calculate_total_steps(arguments_dict):
"""
Get number of in-progress steps in importing process, as shown in the UI.
+
+ There are steps that are general for all sources, but there are steps that are repeated in each source.
+ All of this is taken into account to make the sum
"""
- return len(list(MigrationStep))
+ sources_count = len(arguments_dict.get('sources_pks', 1))
+
+ # STAGING, PARSING, IMPORTING_ASSETS, IMPORTING_STRUCTURE, MAPPING_OLD_TO_NEW, UNSTAGING
+ steps_repeated_count = 6
+
+ return (
+ # All migration steps and subtract the BULK_MIGRATION_PREFIX
+ len(list(MigrationStep)) - 1
+ # We don't want to count these steps again, they will be counted in the operation below.
+ - steps_repeated_count
+ # Each source repeats all the `steps_repeated_count`
+ + steps_repeated_count * sources_count
+ )
@dataclass(frozen=True)
@@ -91,12 +114,15 @@ class _MigrationContext:
"""
Context for the migration process.
"""
- existing_source_to_target_keys: dict[ # Note: It's intended to be mutable to reflect changes during migration.
- UsageKey, list[PublishableEntity]
- ]
+ # Fields that get mutated as we migrate blocks
+ used_component_keys: set[LibraryUsageLocatorV2]
+ used_container_slugs: set[str]
+
+ # Fields that remain constant
+ previous_block_migrations: dict[UsageKey, data.ModulestoreBlockMigrationResult]
target_package_id: int
target_library_key: LibraryLocatorV2
- source_context_key: CourseKey # Note: This includes legacy LibraryLocators, which are sneakily CourseKeys.
+ source_context_key: SourceContextKey
content_by_filename: dict[str, int]
composition_level: CompositionLevel
repeat_handling_strategy: RepeatHandlingStrategy
@@ -104,37 +130,6 @@ class _MigrationContext:
created_by: int
created_at: datetime
- def is_already_migrated(self, source_key: UsageKey) -> bool:
- return source_key in self.existing_source_to_target_keys
-
- def get_existing_target(self, source_key: UsageKey) -> PublishableEntity:
- """
- Get the target entity for a given source key.
-
- If the source key is already migrated, return the FIRST target entity.
- If the source key is not found, raise a KeyError.
- """
- if source_key not in self.existing_source_to_target_keys:
- raise KeyError(f"Source key {source_key} not found in existing source to target keys")
-
- # NOTE: This is a list of PublishableEntities, but we always return the first one.
- return self.existing_source_to_target_keys[source_key][0]
-
- def add_migration(self, source_key: UsageKey, target: PublishableEntity) -> None:
- """Update the context with a new migration (keeps it current)"""
- if source_key not in self.existing_source_to_target_keys:
- self.existing_source_to_target_keys[source_key] = [target]
- else:
- self.existing_source_to_target_keys[source_key].append(target)
-
- def get_existing_target_entity_keys(self, base_key: str) -> set[str]:
- return set(
- publishable_entity.key
- for publishable_entity_list in self.existing_source_to_target_keys.values()
- for publishable_entity in publishable_entity_list
- if publishable_entity.key.startswith(base_key)
- )
-
@property
def should_skip_strategy(self) -> bool:
"""
@@ -157,44 +152,36 @@ def should_fork_strategy(self) -> bool:
return self.repeat_handling_strategy is RepeatHandlingStrategy.Fork
-@shared_task(base=_MigrationTask, bind=True)
-# Note: The decorator @set_code_owner_attribute cannot be used here because the UserTaskMixin
-# does stack inspection and can't handle additional decorators.
-def migrate_from_modulestore(
- self: _MigrationTask,
- *,
- user_id: int,
+@dataclass()
+class _MigrationSourceData:
+ """
+ Data related to a ModulestoreSource
+ """
+ source: models.ModulestoreSource
+ source_root_usage_key: BlockUsageLocator
+ source_version: str | None
+ migration: models.ModulestoreMigration
+ previous_migration: data.ModulestoreMigration | None
+
+
+def _validate_input(
+ status: UserTaskStatus,
source_pk: int,
- target_package_pk: int,
- target_library_key: str,
- target_collection_pk: int,
repeat_handling_strategy: str,
preserve_url_slugs: bool,
composition_level: str,
- forward_source_to_target: bool,
-) -> None:
+ target_library_key: LibraryLocatorV2,
+ target_package: LearningPackage,
+ target_collection: Collection | None,
+) -> _MigrationSourceData | None:
"""
- Import a course or legacy library into a learning package.
-
- Currently, the target learning package must be associated with a V2 content library, but that
- restriction may be loosened in the future as more types of learning packages are developed.
+ Validates and build the source data related to `source_pk`
"""
- # pylint: disable=too-many-statements
- # This is a large function, but breaking it up futher would probably not
- # make it any easier to understand.
-
- set_code_owner_attribute_from_module(__name__)
-
- status: UserTaskStatus = self.status
- status.set_state(MigrationStep.VALIDATING_INPUT.value)
try:
- source = ModulestoreSource.objects.get(pk=source_pk)
- target_package = LearningPackage.objects.get(pk=target_package_pk)
- target_library = get_library(LibraryLocatorV2.from_string(target_library_key))
- target_collection = Collection.objects.get(pk=target_collection_pk) if target_collection_pk else None
- except (ObjectDoesNotExist, InvalidKeyError) as exc:
+ source = models.ModulestoreSource.objects.get(pk=source_pk)
+ except (ObjectDoesNotExist) as exc:
status.fail(str(exc))
- return
+ return None
# The Model is used for Course and Legacy Library
course_index = SplitModulestoreCourseIndex.objects.filter(course_id=source.key).first()
@@ -209,9 +196,19 @@ def migrate_from_modulestore(
f"Not a valid source context key: {source.key}. "
"Source key must reference a course or a legacy library."
)
- return
-
- migration = ModulestoreMigration.objects.create(
+ return None
+
+ # Find the latest successful migration that occurred, if any.
+ # We're careful to do this before creating the new ModulestoreMigration object,
+ # otherwise we would just end up grabbing that by one accident.
+ # ( mypy gets confused by how use next(...) here )
+ previous_migration = next( # type: ignore[call-overload]
+ get_migrations(
+ source.key, target_key=target_library_key, is_failed=False
+ ),
+ None, # default
+ )
+ migration = models.ModulestoreMigration.objects.create(
source=source,
source_version=source_version,
composition_level=composition_level,
@@ -221,21 +218,36 @@ def migrate_from_modulestore(
target_collection=target_collection,
task_status=status,
)
- status.increment_completed_steps()
+ return _MigrationSourceData(
+ source=source,
+ source_root_usage_key=source_root_usage_key,
+ source_version=source_version,
+ migration=migration,
+ previous_migration=previous_migration,
+ )
+
- status.set_state(MigrationStep.CANCELLING_OLD.value)
+def _cancel_old_tasks(
+ source_list: list[models.ModulestoreSource],
+ status: UserTaskStatus,
+ target_package: LearningPackage,
+ migration_ids_to_exclude: list[int],
+) -> None:
+ """
+ Cancel all migration tasks related to the user and the source list
+ """
# In order to prevent a user from accidentally starting a bunch of identical import tasks...
- migrations_to_cancel = ModulestoreMigration.objects.filter(
- # get all Migration tasks by this user with the same source and target
+ migrations_to_cancel = models.ModulestoreMigration.objects.filter(
+ # get all Migration tasks by this user with the same sources and target
task_status__user=status.user,
- source=source,
+ source__in=source_list,
target=target_package,
).select_related('task_status').exclude(
# (excluding that aren't running)
task_status__state__in=(UserTaskStatus.CANCELED, UserTaskStatus.FAILED, UserTaskStatus.SUCCEEDED)
).exclude(
- # (excluding this migration itself)
- id=migration.id
+ # (excluding these migrations themselves)
+ id__in=migration_ids_to_exclude
)
# ... and cancel their tasks and clean away their staged content.
for migration_to_cancel in migrations_to_cancel:
@@ -243,45 +255,41 @@ def migrate_from_modulestore(
migration_to_cancel.task_status.cancel()
if migration_to_cancel.staged_content:
migration_to_cancel.staged_content.delete()
- status.increment_completed_steps()
- status.set_state(MigrationStep.LOADING)
+
+def _load_xblock(
+ status: UserTaskStatus,
+ usage_key: UsageKey,
+) -> XBlock | None:
+ """
+ Loads the Xblock for the given usage_key
+ """
try:
- legacy_root = modulestore().get_item(source_root_usage_key)
+ xblock = modulestore().get_item(usage_key)
except modulestore_exceptions.ItemNotFoundError as exc:
- status.fail(f"Failed to load source item '{source_root_usage_key}' from ModuleStore: {exc}")
- return
- if not legacy_root:
- status.fail(f"Could not find source item '{source_root_usage_key}' in ModuleStore")
- return
- status.increment_completed_steps()
+ status.fail(f"Failed to load source item '{usage_key}' from ModuleStore: {exc}")
+ return None
+ if not xblock:
+ status.fail(f"Could not find source item '{usage_key}' in ModuleStore")
+ return None
+ return xblock
- status.set_state(MigrationStep.STAGING.value)
- staged_content = staging_api.stage_xblock_temporarily(
- block=legacy_root,
- user_id=status.user.pk,
- purpose=CONTENT_STAGING_PURPOSE_TEMPLATE.format(source_key=source.key),
- )
- migration.staged_content = staged_content
- status.increment_completed_steps()
- status.set_state(MigrationStep.PARSING.value)
- parser = etree.XMLParser(strip_cdata=False)
- try:
- root_node = etree.fromstring(staged_content.olx, parser=parser)
- except etree.ParseError as exc:
- status.fail(f"Failed to parse source OLX (from staged content with id = {staged_content.id}): {exc}")
- status.increment_completed_steps()
+def _import_assets(migration: models.ModulestoreMigration) -> dict[str, int]:
+ """
+ Import the assets of the staged content to the migration target
+ """
+ if migration.staged_content is None:
+ return {}
- status.set_state(MigrationStep.IMPORTING_ASSETS.value)
content_by_filename: dict[str, int] = {}
now = datetime.now(tz=timezone.utc)
- for staged_content_file_data in staging_api.get_staged_content_static_files(staged_content.id):
+ for staged_content_file_data in staging_api.get_staged_content_static_files(migration.staged_content.id):
old_path = staged_content_file_data.filename
- file_data = staging_api.get_staged_content_static_file_data(staged_content.id, old_path)
+ file_data = staging_api.get_staged_content_static_file_data(migration.staged_content.id, old_path)
if not file_data:
log.error(
- f"Staged content {staged_content.id} included referenced file {old_path}, "
+ f"Staged content {migration.staged_content.id} included referenced file {old_path}, "
"but no file data was found."
)
continue
@@ -294,90 +302,414 @@ def migrate_from_modulestore(
data=file_data,
created=now,
).id
- status.increment_completed_steps()
+ return content_by_filename
- status.set_state(MigrationStep.IMPORTING_STRUCTURE.value)
- # "key" is locally unique across all PublishableEntities within
- # a given LearningPackage.
- # We use this mapping to ensure that we don't create duplicate
- # PublishableEntities during the migration process for a given LearningPackage.
- existing_source_to_target_keys: dict[UsageKey, list[PublishableEntity]] = {}
- modulestore_blocks = (
- ModulestoreBlockMigration.objects.filter(overall_migration__target=migration.target.id).order_by("source__key")
- )
- existing_source_to_target_keys = {
- source_key: list(block.target for block in group) for source_key, group in groupby(
- modulestore_blocks, key=lambda x: x.source.key)
- }
+def _import_structure(
+ source_data: _MigrationSourceData,
+ target_library: libraries_api.ContentLibraryMetadata,
+ content_by_filename: dict[str, int],
+ root_node: XmlTree,
+ status: UserTaskStatus,
+) -> tuple[t.Any, _MigratedNode]:
+ """
+ Import the staged content structure into the target Learning Core library.
+
+ Args:
+ migration (ModulestoreMigration):
+ The migration record representing the ongoing modulestore-to-learning-core migration.
+ source_data (_MigrationSourceData):
+ Data extracted from the legacy modulestore, including the source root usage key.
+ Use `_validate_input()` to generate this data.
+ target_library (libraries_api.ContentLibraryMetadata):
+ The target library where the new Learning Core content will be created.
+ content_by_filename (dict[str, int]):
+ A mapping between OLX file names and their associated file IDs in the staging area.
+ Use `_import_assets` to generate this content.
+ root_node (XmlTree):
+ The parsed XML tree representing the root of the staged OLX content.
+ status (UserTaskStatus):
+ The user task used to record progress and state updates throughout the import.
+ Returns:
+ tuple[Any, _MigratedNode]:
+ A tuple containing:
+ - The first element (`change_log`): the bulk draft change log generated by
+ `authoring_api.bulk_draft_changes_for`, containing all the imported changes.
+ - The second element (`root_migrated_node`): a `_MigratedNode` object that
+ represents the mapping between the legacy root node and its newly created
+ Learning Core equivalent.
+ """
+ migration = source_data.migration
migration_context = _MigrationContext(
- existing_source_to_target_keys=existing_source_to_target_keys,
- target_package_id=target_package_pk,
+ used_component_keys=set(
+ LibraryUsageLocatorV2(target_library.key, block_type, block_id) # type: ignore[abstract]
+ for block_type, block_id
+ in authoring_api.get_components(migration.target.pk).values_list(
+ "component_type__name", "local_key"
+ )
+ ),
+ used_container_slugs=set(
+ authoring_api.get_containers(
+ migration.target.pk
+ ).values_list("publishable_entity__key", flat=True)
+ ),
+ previous_block_migrations=(
+ get_migration_blocks(source_data.previous_migration.pk)
+ if source_data.previous_migration
+ else {}
+ ),
+ target_package_id=migration.target.pk,
target_library_key=target_library.key,
- source_context_key=source_root_usage_key.course_key,
+ source_context_key=source_data.source_root_usage_key.course_key,
content_by_filename=content_by_filename,
- composition_level=CompositionLevel(composition_level),
- repeat_handling_strategy=RepeatHandlingStrategy(repeat_handling_strategy),
- preserve_url_slugs=preserve_url_slugs,
+ composition_level=CompositionLevel(migration.composition_level),
+ repeat_handling_strategy=RepeatHandlingStrategy(migration.repeat_handling_strategy),
+ preserve_url_slugs=migration.preserve_url_slugs,
created_by=status.user_id,
created_at=datetime.now(timezone.utc),
)
-
with authoring_api.bulk_draft_changes_for(migration.target.id) as change_log:
root_migrated_node = _migrate_node(
context=migration_context,
source_node=root_node,
)
change_log.save()
- migration.change_log = change_log
- status.increment_completed_steps()
+ return change_log, root_migrated_node
- status.set_state(MigrationStep.UNSTAGING.value)
- staged_content.delete()
- status.increment_completed_steps()
- _create_migration_artifacts_incrementally(
- root_migrated_node=root_migrated_node,
- source=source,
- migration=migration,
- status=status,
+def _forward_content(source_data: _MigrationSourceData) -> None:
+ """
+ Forwarding legacy content to migrated content
+ """
+ block_migrations = models.ModulestoreBlockMigration.objects.filter(overall_migration=source_data.migration)
+ block_sources_to_block_migrations = {
+ block_migration.source: block_migration for block_migration in block_migrations
+ }
+ for block_source, block_migration in block_sources_to_block_migrations.items():
+ block_source.forwarded = block_migration
+ block_source.save()
+
+ source_data.source.forwarded = source_data.migration
+ source_data.source.save()
+
+
+def _populate_collection(user_id: int, migration: models.ModulestoreMigration) -> None:
+ """
+ Assigning imported items to the specified collection in the migration
+ """
+ if migration.target_collection is None:
+ return
+
+ block_target_pks: list[int] = list(
+ models.ModulestoreBlockMigration.objects.filter(
+ overall_migration=migration
+ ).values_list('target_id', flat=True)
)
+ if block_target_pks:
+ authoring_api.add_to_collection(
+ learning_package_id=migration.target.pk,
+ key=migration.target_collection.key,
+ entities_qset=PublishableEntity.objects.filter(id__in=block_target_pks),
+ created_by=user_id,
+ )
+ else:
+ log.warning("No target entities found to add to collection")
- block_migrations = ModulestoreBlockMigration.objects.filter(overall_migration=migration)
- status.increment_completed_steps()
- status.set_state(MigrationStep.FORWARDING.value)
- if forward_source_to_target:
- block_sources_to_block_migrations = {
- block_migration.source: block_migration for block_migration in block_migrations
- }
- for block_source, block_migration in block_sources_to_block_migrations.items():
- block_source.forwarded = block_migration
- block_source.save()
-
- source.forwarded = migration
- source.save()
- status.increment_completed_steps()
+def _create_collection(library_key: LibraryLocatorV2, title: str) -> Collection:
+ """
+ Creates a collection in the given library
+
+ If there's a collection with the same key, try again, adding the attempt number at the end.
+ The same is true for the title.
+ """
+ key = slugify(title)
+ collection: Collection | None = None
+ attempt = 0
+ created_at = strftime_localized(datetime.now(timezone.utc), DEFAULT_DATE_TIME_FORMAT)
+ description = f"{_('This collection contains content migrated from a legacy library on')}: {created_at}"
+ while not collection:
+ modified_key = key if attempt == 0 else key + '-' + str(attempt)
+ try:
+ # Add transaction here to avoid TransactionManagementError on retry
+ with transaction.atomic():
+ collection = libraries_api.create_library_collection(
+ library_key=library_key,
+ collection_key=modified_key,
+ title=f"{title}{f'_{attempt}' if attempt > 0 else ''}",
+ description=description,
+ )
+ except libraries_api.LibraryCollectionAlreadyExists:
+ attempt += 1
+ return collection
+
+
+def _set_migrations_to_fail(source_data_list: list[_MigrationSourceData]):
+ """
+ Set and save all migrations in `source_data_list` as failed
+ """
+ for source_data in source_data_list:
+ source_data.migration.is_failed = True
- status.set_state(MigrationStep.POPULATING_COLLECTION.value)
- if target_collection:
- block_target_pks: list[int] = list(
- ModulestoreBlockMigration.objects.filter(
- overall_migration=migration
- ).values_list('target_id', flat=True)
+ models.ModulestoreMigration.objects.bulk_update(
+ [x.migration for x in source_data_list],
+ ["is_failed"],
+ )
+
+
+@shared_task(base=_BulkMigrationTask, bind=True)
+# Note: The decorator @set_code_owner_attribute cannot be used here because the UserTaskMixin
+# does stack inspection and can't handle additional decorators.
+def bulk_migrate_from_modulestore(
+ self: _BulkMigrationTask,
+ *,
+ user_id: int,
+ sources_pks: list[int],
+ target_library_key: str,
+ target_collection_pks: list[int | None],
+ create_collections: bool = False,
+ repeat_handling_strategy: str,
+ preserve_url_slugs: bool,
+ composition_level: str,
+ forward_source_to_target: bool | None,
+) -> None:
+ """
+ Import multiple legacy courses or libraries into a single V2 library.
+
+ The bulk migration maintains **one unified status record** that tracks progress across
+ all included sources. This simplifies monitoring, since the client only needs to observe
+ one task state.
+ Each source item (course or library) still creates its own `ModulestoreMigration`
+ database record, but all of them share the same parent task (`UserTaskStatus`).
+ If any sub-migration fails (for example, due to invalid OLX or missing assets),
+ the bulk migration **marks the entire task as failed** — there is no partial success.
+
+ Args:
+ self (_BulkMigrationTask):
+ The Celery task instance that wraps the user task logic.
+ user_id (int):
+ The ID of the user initiating the migration.
+ sources_pks (list[int]):
+ Primary keys of the legacy modulestore sources to migrate.
+ target_library_key (str):
+ Key of the V2 library that will receive the imported content.
+ target_collection_pks (list[int | None]):
+ Optional list of target collection IDs corresponding to each source.
+ create_collections (bool):
+ Whether to automatically create new collections when none exist.
+ repeat_handling_strategy (str):
+ Strategy to handle repeated imports of the same content.
+ preserve_url_slugs (bool):
+ Whether to preserve existing XBlock URL slugs during import.
+ composition_level (str):
+ Composition level at which content should be imported (e.g. course, section).
+ forward_source_to_target (bool | None)
+ Whether to forward legacy content to its migrated equivalent after import.
+ If unspecified (None), then forward legacy content for a source if and only
+ if it's that source's first migration.
+
+ See Also:
+ - `migrate_from_modulestore`: Single-source migration equivalent.
+ - API docs: `/api/cms/v1/migrations/bulk/` for REST behavior and responses.
+ """
+ # pylint: disable=too-many-statements
+ # This is a large function, but breaking it up futher would probably not
+ # make it any easier to understand.
+
+ set_code_owner_attribute_from_module(__name__)
+ status: UserTaskStatus = self.status
+
+ # Validating input
+ status.set_state(MigrationStep.VALIDATING_INPUT.value)
+ target_collection_list: list[Collection | None] = []
+
+ try:
+ target_library_locator = LibraryLocatorV2.from_string(target_library_key)
+ target_library = get_library(target_library_locator)
+ if target_library.learning_package_id is None:
+ raise ValueError("Target library has no associated learning package.")
+
+ target_package = LearningPackage.objects.get(pk=target_library.learning_package_id)
+
+ if target_collection_pks:
+ for target_collection_pk in target_collection_pks:
+ target_collection_list.append(
+ Collection.objects.get(pk=target_collection_pk) if target_collection_pk else None
+ )
+ except (ObjectDoesNotExist, InvalidKeyError, ValueError) as exc:
+ status.fail(str(exc))
+ return
+
+ source_data_list: list[_MigrationSourceData] = []
+
+ for i in range(len(sources_pks)):
+ source_data = _validate_input(
+ status,
+ sources_pks[i],
+ repeat_handling_strategy,
+ preserve_url_slugs,
+ composition_level,
+ target_library_locator,
+ target_package,
+ target_collection_list[i] if target_collection_list else None,
)
- if block_target_pks:
- authoring_api.add_to_collection(
- learning_package_id=target_package_pk,
- key=target_collection.key,
- entities_qset=PublishableEntity.objects.filter(id__in=block_target_pks),
- created_by=user_id,
- )
- else:
- log.warning("No target entities found to add to collection")
+ if source_data is None:
+ # Fail
+ return
+ source_data_list.append(source_data)
+
status.increment_completed_steps()
+ try: # pylint: disable=too-many-nested-blocks
+ # Cancelling old tasks
+ status.set_state(MigrationStep.CANCELLING_OLD.value)
+ _cancel_old_tasks(
+ [x.source for x in source_data_list],
+ status,
+ target_package,
+ [migration.id for migration in [x.migration for x in source_data_list]],
+ )
+ status.increment_completed_steps()
+
+ # Loading legacy blocks
+ status.set_state(MigrationStep.LOADING)
+ legacy_root_list: list[XBlock] = []
+ for source_data in source_data_list:
+ legacy_root = _load_xblock(status, source_data.source_root_usage_key)
+ if legacy_root is None:
+ # Fail
+ _set_migrations_to_fail(source_data_list)
+ return
+ legacy_root_list.append(legacy_root)
+ status.increment_completed_steps()
+
+ for i, source_pk in enumerate(sources_pks):
+ source_data = source_data_list[i]
+ try:
+ with transaction.atomic():
+ # Start migration for `source_pk`
+ # Staging legacy blocks
+ status.set_state(
+ f"{MigrationStep.STAGING.BULK_MIGRATION_PREFIX} ({source_pk}): {MigrationStep.STAGING.value}"
+ )
+ staged_content = staging_api.stage_xblock_temporarily(
+ block=legacy_root_list[i],
+ user_id=status.user.pk,
+ purpose=CONTENT_STAGING_PURPOSE_TEMPLATE.format(source_key=source_pk),
+ )
+ source_data.migration.staged_content = staged_content
+ status.increment_completed_steps()
+
+ # Parsing OLX
+ status.set_state(
+ f"{MigrationStep.STAGING.BULK_MIGRATION_PREFIX} ({source_pk}): {MigrationStep.PARSING.value}"
+ )
+ parser = etree.XMLParser(strip_cdata=False)
+ root_node = etree.fromstring(staged_content.olx, parser=parser)
+ status.increment_completed_steps()
+
+ # Importing assets
+ status.set_state(
+ f"{MigrationStep.STAGING.BULK_MIGRATION_PREFIX} ({source_pk}): "
+ f"{MigrationStep.IMPORTING_ASSETS.value}"
+ )
+ content_by_filename = _import_assets(source_data.migration)
+ status.increment_completed_steps()
+
+ # Importing structure of the legacy block
+ status.set_state(
+ f"{MigrationStep.STAGING.BULK_MIGRATION_PREFIX} ({source_pk}): "
+ f"{MigrationStep.IMPORTING_STRUCTURE.value}"
+ )
+ change_log, root_migrated_node = _import_structure(
+ source_data=source_data,
+ target_library=target_library,
+ content_by_filename=content_by_filename,
+ root_node=root_node,
+ status=status,
+ )
+ source_data.migration.change_log = change_log
+ source_data.migration.save() # @@TODO keep or nah?
+ status.increment_completed_steps()
+
+ status.set_state(
+ f"{MigrationStep.STAGING.BULK_MIGRATION_PREFIX} ({source_pk}): {MigrationStep.UNSTAGING.value}"
+ )
+ staged_content.delete()
+ status.increment_completed_steps()
+
+ _create_migration_artifacts_incrementally(
+ root_migrated_node=root_migrated_node,
+ source=source_data.source,
+ migration=source_data.migration,
+ status=status,
+ source_pk=source_pk,
+ )
+ status.increment_completed_steps()
+ except Exception as _exc: # pylint: disable=broad-exception-caught
+ log.exception("Failed: {source_data.migration}")
+ # Mark this library as failed, migration of other libraries can continue
+ # If this case occurs and the migration ends without any further issues,
+ # the bulk migration status is success,
+ # TODO: add an intermediate status such as 'partially satisfactory'
+ source_data.migration.is_failed = True
+
+ # Forwarding legacy content to migrated content
+ status.set_state(MigrationStep.FORWARDING.value)
+ for source_data in source_data_list:
+ if forward_source_to_target is False:
+ continue # Explicitly requested not to forward.
+ if forward_source_to_target is None and source_data.source.forwarded:
+ # Unspecified whether or not to forward.
+ # So, forward iff there was no previous existing successful migration with forwarding.
+ continue
+ if source_data.migration.is_failed:
+ # Don't forward failed migrations.
+ continue
+ _forward_content(source_data)
+ status.increment_completed_steps()
+
+ # Populating collections
+ status.set_state(MigrationStep.POPULATING_COLLECTION.value)
+ for i, source_data in enumerate(source_data_list):
+ migration = source_data.migration
+ if migration.is_failed:
+ continue
+ if migration.target_collection is None and not create_collections:
+ continue
+ if migration.target_collection is None:
+ existing_collection_to_use: Collection | None = None
+ # For Fork strategy: Create an new collection every time.
+ # For Update and Skip strategies: Update an existing collection if possible.
+ if migration.repeat_handling_strategy != RepeatHandlingStrategy.Fork.value:
+ if source_data.previous_migration:
+ if previous_collection_slug := source_data.previous_migration.target_collection_slug:
+ try:
+ existing_collection_to_use = authoring_api.get_collection(
+ target_package.id, previous_collection_slug
+ )
+ except Collection.DoesNotExist:
+ # Collection no longer exists.
+ pass
+ migration.target_collection = (
+ existing_collection_to_use or
+ _create_collection(library_key=target_library_locator, title=legacy_root_list[i].display_name)
+ )
+ _populate_collection(user_id, migration)
+ models.ModulestoreMigration.objects.bulk_update(
+ [x.migration for x in source_data_list],
+ ["target_collection", "is_failed"],
+ )
+ status.increment_completed_steps()
+ except Exception as exc: # pylint: disable=broad-exception-caught
+ # If there is an exception in this block, all migrations fail.
+ log.exception("Modulestore migrations failed")
+ status.fail(str(exc))
+
+
+SourceToTarget = tuple[UsageKey, PublishableEntityVersion | None, str | None]
+
@dataclass(frozen=True)
class _MigratedNode:
@@ -388,10 +720,10 @@ class _MigratedNode:
This happens, particularly, if the node is above the requested composition level
but has descendents which are at or below that level.
"""
- source_to_target: tuple[UsageKey, PublishableEntityVersion] | None
+ source_to_target: SourceToTarget | None
children: list[_MigratedNode]
- def all_source_to_target_pairs(self) -> t.Iterable[tuple[UsageKey, PublishableEntityVersion]]:
+ def all_source_to_target_pairs(self) -> t.Iterable[SourceToTarget]:
"""
Get all source_key->target_ver pairs via a pre-order traversal.
"""
@@ -449,13 +781,13 @@ def _migrate_node(
)
for source_node_child in source_node.getchildren()
]
- source_to_target: tuple[UsageKey, PublishableEntityVersion] | None = None
+ source_to_target: SourceToTarget | None = None
if should_migrate_node:
source_olx = etree.tostring(source_node).decode('utf-8')
if source_block_id := source_node.get('url_name'):
source_key: UsageKey = context.source_context_key.make_usage_key(source_node.tag, source_block_id)
title = source_node.get('display_name', source_block_id)
- target_entity_version = (
+ target_entity_version, reason = (
_migrate_container(
context=context,
source_key=source_key,
@@ -464,7 +796,7 @@ def _migrate_node(
children=[
migrated_child.source_to_target[1]
for migrated_child in migrated_children if
- migrated_child.source_to_target
+ migrated_child.source_to_target and migrated_child.source_to_target[1]
],
)
if container_type else
@@ -475,9 +807,18 @@ def _migrate_node(
title=title,
)
)
- if target_entity_version:
- source_to_target = (source_key, target_entity_version)
- context.add_migration(source_key, target_entity_version.entity)
+ if container_type is None and target_entity_version is None and reason is not None:
+ # Currently, components with children are not supported
+ children_length = len(source_node.getchildren())
+ if children_length:
+ reason += (
+ ngettext(
+ ' It has {count} children block.',
+ ' It has {count} children blocks.',
+ children_length,
+ )
+ ).format(count=children_length)
+ source_to_target = (source_key, target_entity_version, reason)
else:
log.warning(
f"Cannot migrate node from {context.source_context_key} to {context.target_library_key} "
@@ -493,12 +834,14 @@ def _migrate_container(
container_type: ContainerType,
title: str,
children: list[PublishableEntityVersion],
-) -> PublishableEntityVersion:
+) -> tuple[PublishableEntityVersion, str | None]:
"""
Create, update, or replace a container in a library based on a source key and children.
(We assume that the destination is a library rather than some other future kind of learning
- package, but let's keep than an internal assumption.)
+ package, but let's keep than an internal assumption.)
+ For now this returns None value for unsupported_reason as second value of tuple as we
+ don't have any concrete condition where a container cannot be imported/migrated.
"""
target_key = _get_distinct_target_container_key(
context,
@@ -530,8 +873,9 @@ def _migrate_container(
return PublishableEntityVersion.objects.get(
entity_id=container.container_pk,
version_num=container.draft_version_num,
- )
- return authoring_api.create_next_container_version(
+ ), None
+
+ container_publishable_entity_version = authoring_api.create_next_container_version(
container.container_pk,
title=title,
entity_rows=[
@@ -543,6 +887,18 @@ def _migrate_container(
container_version_cls=container_type.container_model_classes[1],
).publishable_entity_version
+ # Publish the container
+ # Call post publish events synchronously to avoid
+ # an error when calling `wait_for_post_publish_events`
+ # inside a celery task.
+ libraries_api.publish_container_changes(
+ container.container_key,
+ context.created_by,
+ call_post_publish_events_sync=True,
+ )
+ context.used_container_slugs.add(container.container_key.container_id)
+ return container_publishable_entity_version, None
+
def _migrate_component(
*,
@@ -550,7 +906,7 @@ def _migrate_component(
source_key: UsageKey,
olx: str,
title: str,
-) -> PublishableEntityVersion | None:
+) -> tuple[PublishableEntityVersion | None, str | None]:
"""
Create, update, or replace a component in a library based on a source key and OLX.
@@ -583,7 +939,10 @@ def _migrate_component(
)
except libraries_api.IncompatibleTypesError as e:
log.error(f"Error validating block for library {context.target_library_key}: {e}")
- return None
+ return None, str(e)
+ except PluginMissingError as e:
+ log.error(f"Block type not supported in {context.target_library_key}: {e}")
+ return None, f"Invalid block type: {e}"
component = authoring_api.create_component(
context.target_package_id,
component_type=component_type,
@@ -594,7 +953,7 @@ def _migrate_component(
# Component existed and we do not replace it and it is not deleted previously
if component_existed and not component_deleted and context.should_skip_strategy:
- return component.versioning.draft.publishable_entity_version
+ return component.versioning.draft.publishable_entity_version, None
# If component existed and was deleted or we have to replace the current version
# Create the new component version for it
@@ -607,7 +966,14 @@ def _migrate_component(
authoring_api.create_component_version_content(
component_version.pk, content_pk, key=new_path
)
- return component_version.publishable_entity_version
+
+ # Publish the component
+ libraries_api.publish_component_changes(target_key, context.created_by)
+ context.used_component_keys.add(target_key)
+ return component_version.publishable_entity_version, None
+
+
+_MAX_UNIQUE_SLUG_ATTEMPTS = 1000
def _get_distinct_target_container_key(
@@ -617,39 +983,36 @@ def _get_distinct_target_container_key(
title: str,
) -> LibraryContainerLocator:
"""
- Find a unique key for block_id by appending a unique identifier if necessary.
-
- Args:
- context (_MigrationContext): The migration context.
- source_key (UsageKey): The source key.
- container_type (ContainerType): The container type.
- title (str): The title.
-
- Returns:
- LibraryContainerLocator: The target container key.
+ Figure out the appropriate target container for this structural block.
"""
- # Check if we already processed this block and we are not forking. If we are forking, we will
- # want a new target key.
- if context.is_already_migrated(source_key) and not context.should_fork_strategy:
- existing_version = context.get_existing_target(source_key)
-
- return LibraryContainerLocator(
- context.target_library_key,
- container_type.value,
- existing_version.key
- )
+ # If we're not forking, then check if this block was part of our past migration.
+ # (If we are forking, we will always want a new target key).
+ if not context.should_fork_strategy:
+ if previous_block_migration := context.previous_block_migrations.get(source_key):
+ if isinstance(previous_block_migration, data.ModulestoreBlockMigrationSuccess):
+ if isinstance(previous_block_migration.target_key, LibraryContainerLocator):
+ return previous_block_migration.target_key
# Generate new unique block ID
base_slug = (
source_key.block_id
if context.preserve_url_slugs
else (slugify(title) or source_key.block_id)
)
- unique_slug = _find_unique_slug(context, base_slug)
-
- return LibraryContainerLocator(
- context.target_library_key,
- container_type.value,
- unique_slug
+ # Use base base slug if available
+ if base_slug not in context.used_container_slugs:
+ return LibraryContainerLocator(
+ context.target_library_key, container_type.value, base_slug
+ )
+ # Try numbered variations until we find one that doesn't exist
+ for i in range(1, _MAX_UNIQUE_SLUG_ATTEMPTS + 1):
+ candidate_slug = f"{base_slug}_{i}"
+ if candidate_slug not in context.used_container_slugs:
+ return LibraryContainerLocator(
+ context.target_library_key, container_type.value, candidate_slug
+ )
+ # It would be extremely unlikely for us to run out of attempts
+ raise RuntimeError(
+ f"Unable to find unique slug after {_MAX_UNIQUE_SLUG_ATTEMPTS} attempts for base: {base_slug}"
)
@@ -660,98 +1023,45 @@ def _get_distinct_target_usage_key(
title: str,
) -> LibraryUsageLocatorV2:
"""
- Find a unique key for block_id by appending a unique identifier if necessary.
-
- Args:
- context: The migration context
- source_key: The original usage key from the source
- component_type: The component type string
- olx: The OLX content of the component
-
- Returns:
- A unique LibraryUsageLocatorV2 for the target
-
- Raises:
- ValueError: If source_key is invalid
- """
- # Check if we already processed this block and we are not forking. If we are forking, we will
- # want a new target key.
- if context.is_already_migrated(source_key) and not context.should_fork_strategy:
- log.debug(f"Block {source_key} already exists, reusing first existing target")
- existing_target = context.get_existing_target(source_key)
- block_id = existing_target.component.local_key
-
- # mypy thinks LibraryUsageLocatorV2 is abstract. It's not.
- return LibraryUsageLocatorV2( # type: ignore[abstract]
- context.target_library_key,
- source_key.block_type,
- block_id
- )
-
+ Figure out the appropriate target component for this block.
+ """
+ # If we're not forking, then check if this block was part of our past migration.
+ # (If we are forking, we will always want a new target key).
+ if not context.should_fork_strategy:
+ if previous_block_migration := context.previous_block_migrations.get(source_key):
+ if isinstance(previous_block_migration, data.ModulestoreBlockMigrationSuccess):
+ if isinstance(previous_block_migration.target_key, LibraryUsageLocatorV2):
+ return previous_block_migration.target_key
# Generate new unique block ID
base_slug = (
source_key.block_id
if context.preserve_url_slugs
else (slugify(title) or source_key.block_id)
)
- unique_slug = _find_unique_slug(context, base_slug, component_type)
-
- # mypy thinks LibraryUsageLocatorV2 is abstract. It's not.
- return LibraryUsageLocatorV2( # type: ignore[abstract]
- context.target_library_key,
- source_key.block_type,
- unique_slug
+ # Use base base slug if available
+ base_key = LibraryUsageLocatorV2( # type: ignore[abstract]
+ context.target_library_key, component_type.name, base_slug
)
-
-
-def _find_unique_slug(
- context: _MigrationContext,
- base_slug: str,
- component_type: ComponentType | None = None,
- max_attempts: int = 1000
-) -> str:
- """
- Find a unique slug by appending incrementing numbers if necessary.
- Using batch querying to avoid multiple database roundtrips.
-
- Args:
- component_type: The component type to check against
- base_slug: The base slug to make unique
- max_attempts: Maximum number of attempts to prevent infinite loops
-
- Returns:
- A unique slug string
-
- Raises:
- RuntimeError: If unable to find unique slug within max_attempts
- """
- if not component_type:
- base_key = base_slug
- else:
- base_key = f"{component_type}:{base_slug}"
-
- existing_publishable_entity_keys = context.get_existing_target_entity_keys(base_key)
-
- # Check if base slug is available
- if base_key not in existing_publishable_entity_keys:
- return base_slug
-
+ if base_key not in context.used_component_keys:
+ return base_key
# Try numbered variations until we find one that doesn't exist
- for i in range(1, max_attempts + 1):
+ for i in range(1, _MAX_UNIQUE_SLUG_ATTEMPTS + 1):
candidate_slug = f"{base_slug}_{i}"
- candidate_key = f"{component_type}:{candidate_slug}" if component_type else candidate_slug
-
- if candidate_key not in existing_publishable_entity_keys:
- return candidate_slug
-
- raise RuntimeError(f"Unable to find unique slug after {max_attempts} attempts for base: {base_slug}")
+ candidate_key = LibraryUsageLocatorV2( # type: ignore[abstract]
+ context.target_library_key, component_type.name, candidate_slug
+ )
+ if candidate_key not in context.used_component_keys:
+ return candidate_key
+ # It would be extremely unlikely for us to run out of attempts
+ raise RuntimeError(f"Unable to find unique slug after {_MAX_UNIQUE_SLUG_ATTEMPTS} attempts for base: {base_slug}")
def _create_migration_artifacts_incrementally(
root_migrated_node: _MigratedNode,
- source: ModulestoreSource,
- migration: ModulestoreMigration,
- status: UserTaskStatus
+ source: models.ModulestoreSource,
+ migration: models.ModulestoreMigration,
+ status: UserTaskStatus,
+ source_pk: int | None = None,
) -> None:
"""
Create ModulestoreBlockSource and ModulestoreBlockMigration objects incrementally.
@@ -760,20 +1070,43 @@ def _create_migration_artifacts_incrementally(
total_nodes = len(nodes)
processed = 0
- for source_usage_key, target_version in root_migrated_node.all_source_to_target_pairs():
- block_source, _ = ModulestoreBlockSource.objects.get_or_create(
+ # Load a mapping from each modified entity's primary key
+ # to the primary key of the changelog record that captures its modification.
+ # This will not include any blocks whose migration failed to create a target entity.
+ entity_pks_to_change_log_record_pks: dict[int, int] = dict(
+ migration.change_log.records.values_list("entity_id", "id")
+ ) if migration.change_log else {}
+
+ for source_usage_key, target_version, unsupported_reason in root_migrated_node.all_source_to_target_pairs():
+ block_source, _ = models.ModulestoreBlockSource.objects.get_or_create(
overall_source=source,
key=source_usage_key
)
-
- ModulestoreBlockMigration.objects.create(
- overall_migration=migration,
- source=block_source,
- target_id=target_version.entity_id,
- )
+ # target_entity_pk should be None iff the block migration failed
+ target_entity_pk: int | None = target_version.entity_id if target_version else None
+
+ change_log_record_pk = entity_pks_to_change_log_record_pks.get(target_entity_pk) if target_entity_pk else None
+ # Only create a migration artifact for this source block if:
+ # (a) we have a record of a change occuring, or
+ # (b) it failed.
+ # If neither a nor b are true, then this source block was skipped.
+ if change_log_record_pk or unsupported_reason:
+ models.ModulestoreBlockMigration.objects.create(
+ overall_migration=migration,
+ source=block_source,
+ target_id=target_entity_pk,
+ change_log_record_id=change_log_record_pk,
+ unsupported_reason=unsupported_reason,
+ )
processed += 1
if processed % 10 == 0 or processed == total_nodes:
- status.set_state(
- f"{MigrationStep.MAPPING_OLD_TO_NEW.value} ({processed}/{total_nodes})"
- )
+ if source_pk:
+ status.set_state(
+ f"{MigrationStep.STAGING.BULK_MIGRATION_PREFIX} ({source_pk}): "
+ f"{MigrationStep.MAPPING_OLD_TO_NEW.value} ({processed}/{total_nodes})"
+ )
+ else:
+ status.set_state(
+ f"{MigrationStep.MAPPING_OLD_TO_NEW.value} ({processed}/{total_nodes})"
+ )
diff --git a/cms/djangoapps/modulestore_migrator/tests/test_api.py b/cms/djangoapps/modulestore_migrator/tests/test_api.py
index 13e7e7685d9c..c9e2fc3b587b 100644
--- a/cms/djangoapps/modulestore_migrator/tests/test_api.py
+++ b/cms/djangoapps/modulestore_migrator/tests/test_api.py
@@ -3,11 +3,10 @@
"""
import pytest
-from opaque_keys.edx.locator import LibraryLocatorV2
+from opaque_keys.edx.locator import LibraryLocator, LibraryLocatorV2
from openedx_learning.api import authoring as authoring_api
from organizations.tests.factories import OrganizationFactory
-from cms.djangoapps.contentstore.tests.test_libraries import LibraryTestCase
from cms.djangoapps.modulestore_migrator import api
from cms.djangoapps.modulestore_migrator.data import CompositionLevel, RepeatHandlingStrategy
from cms.djangoapps.modulestore_migrator.models import ModulestoreMigration
@@ -15,29 +14,50 @@
from common.djangoapps.student.tests.factories import UserFactory
from openedx.core.djangoapps.content_libraries import api as lib_api
+from xmodule.modulestore.tests.utils import ModuleStoreTestCase
from xmodule.modulestore.tests.factories import BlockFactory, LibraryFactory
@pytest.mark.django_db
-class TestModulestoreMigratorAPI(LibraryTestCase):
+class TestModulestoreMigratorAPI(ModuleStoreTestCase):
"""
Test cases for the modulestore migrator API.
"""
def setUp(self):
super().setUp()
-
- self.organization = OrganizationFactory()
- self.lib_key_v2 = LibraryLocatorV2.from_string(
- f"lib:{self.organization.short_name}:test-key"
- )
- lib_api.create_library(
- org=self.organization,
- slug=self.lib_key_v2.slug,
- title="Test Library",
- )
- self.library_v2 = lib_api.ContentLibrary.objects.get(slug=self.lib_key_v2.slug)
- self.learning_package = self.library_v2.learning_package
+ self.user = UserFactory(password=self.user_password, is_staff=True)
+ self.organization = OrganizationFactory(name="My Org", short_name="myorg")
+ self.lib_key_v1 = LibraryLocator.from_string("library-v1:myorg+old")
+ LibraryFactory.create(org="myorg", library="old", display_name="Old Library", modulestore=self.store)
+ self.lib_key_v2_1 = LibraryLocatorV2.from_string("lib:myorg:1")
+ self.lib_key_v2_2 = LibraryLocatorV2.from_string("lib:myorg:2")
+ lib_api.create_library(org=self.organization, slug="1", title="Test Library 1")
+ lib_api.create_library(org=self.organization, slug="2", title="Test Library 2")
+ self.library_v2_1 = lib_api.ContentLibrary.objects.get(slug="1")
+ self.library_v2_2 = lib_api.ContentLibrary.objects.get(slug="2")
+ self.learning_package = self.library_v2_1.learning_package
+ self.learning_package_2 = self.library_v2_2.learning_package
+ self.source_unit_keys = [
+ BlockFactory.create(
+ display_name=f"Unit {c}",
+ category="vertical",
+ location=self.lib_key_v1.make_usage_key("vertical", c),
+ parent_location=self.lib_key_v1.make_usage_key("library", "library"),
+ user_id=self.user.id, publish_item=False,
+ ).usage_key for c in ["X", "Y", "Z"]
+ ]
+ self.source_html_keys = [
+ BlockFactory.create(
+ display_name=f"HTML {c}",
+ category="html",
+ location=self.lib_key_v1.make_usage_key("html", c),
+ parent_location=self.lib_key_v1.make_usage_key("vertical", c),
+ user_id=self.user.id, publish_item=False,
+ ).usage_key for c in ["X", "Y", "Z"]
+ ]
+ # We load this last so that it has an updated list of children.
+ self.lib_v1 = self.store.get_library(self.lib_key_v1)
def test_start_migration_to_library(self):
"""
@@ -49,10 +69,10 @@ def test_start_migration_to_library(self):
api.start_migration_to_library(
user=user,
source_key=source.key,
- target_library_key=self.library_v2.library_key,
+ target_library_key=self.library_v2_1.library_key,
target_collection_slug=None,
- composition_level=CompositionLevel.Component.value,
- repeat_handling_strategy=RepeatHandlingStrategy.Skip.value,
+ composition_level=CompositionLevel.Component,
+ repeat_handling_strategy=RepeatHandlingStrategy.Skip,
preserve_url_slugs=True,
forward_source_to_target=False,
)
@@ -67,6 +87,45 @@ def test_start_migration_to_library(self):
assert modulestoremigration.task_status is not None
assert modulestoremigration.task_status.user == user
+ def test_start_bulk_migration_to_library(self):
+ """
+ Test that the API can start a bulk migration to a library.
+ """
+ source = ModulestoreSourceFactory()
+ source_2 = ModulestoreSourceFactory()
+ user = UserFactory()
+
+ api.start_bulk_migration_to_library(
+ user=user,
+ source_key_list=[source.key, source_2.key],
+ target_library_key=self.library_v2_1.library_key,
+ target_collection_slug_list=None,
+ composition_level=CompositionLevel.Component,
+ repeat_handling_strategy=RepeatHandlingStrategy.Skip,
+ preserve_url_slugs=True,
+ forward_source_to_target=False,
+ )
+
+ modulestoremigration = ModulestoreMigration.objects.get(source=source)
+ assert modulestoremigration.source.key == source.key
+ assert (
+ modulestoremigration.composition_level == CompositionLevel.Component.value
+ )
+ assert modulestoremigration.repeat_handling_strategy == RepeatHandlingStrategy.Skip.value
+ assert modulestoremigration.preserve_url_slugs is True
+ assert modulestoremigration.task_status is not None
+ assert modulestoremigration.task_status.user == user
+
+ modulestoremigration_2 = ModulestoreMigration.objects.get(source=source_2)
+ assert modulestoremigration_2.source.key == source_2.key
+ assert (
+ modulestoremigration_2.composition_level == CompositionLevel.Component.value
+ )
+ assert modulestoremigration_2.repeat_handling_strategy == RepeatHandlingStrategy.Skip.value
+ assert modulestoremigration_2.preserve_url_slugs is True
+ assert modulestoremigration_2.task_status is not None
+ assert modulestoremigration_2.task_status.user == user
+
def test_start_migration_to_library_with_collection(self):
"""
Test that the API can start a migration to a library with a target collection.
@@ -86,10 +145,10 @@ def test_start_migration_to_library_with_collection(self):
api.start_migration_to_library(
user=user,
source_key=source.key,
- target_library_key=self.library_v2.library_key,
+ target_library_key=self.library_v2_1.library_key,
target_collection_slug=collection_key,
- composition_level=CompositionLevel.Component.value,
- repeat_handling_strategy=RepeatHandlingStrategy.Skip.value,
+ composition_level=CompositionLevel.Component,
+ repeat_handling_strategy=RepeatHandlingStrategy.Skip,
preserve_url_slugs=True,
forward_source_to_target=False,
)
@@ -115,9 +174,9 @@ def test_start_migration_to_library_with_strategy_skip(self):
api.start_migration_to_library(
user=user,
source_key=source.key,
- target_library_key=self.library_v2.library_key,
- composition_level=CompositionLevel.Component.value,
- repeat_handling_strategy=RepeatHandlingStrategy.Skip.value,
+ target_library_key=self.library_v2_1.library_key,
+ composition_level=CompositionLevel.Component,
+ repeat_handling_strategy=RepeatHandlingStrategy.Skip,
preserve_url_slugs=True,
forward_source_to_target=False,
)
@@ -125,7 +184,7 @@ def test_start_migration_to_library_with_strategy_skip(self):
modulestoremigration = ModulestoreMigration.objects.get()
assert modulestoremigration.repeat_handling_strategy == RepeatHandlingStrategy.Skip.value
- migrated_components = lib_api.get_library_components(self.library_v2.library_key)
+ migrated_components = lib_api.get_library_components(self.library_v2_1.library_key)
assert len(migrated_components) == 1
# Update the block, changing its name
@@ -136,9 +195,9 @@ def test_start_migration_to_library_with_strategy_skip(self):
api.start_migration_to_library(
user=user,
source_key=source.key,
- target_library_key=self.library_v2.library_key,
- composition_level=CompositionLevel.Component.value,
- repeat_handling_strategy=RepeatHandlingStrategy.Skip.value,
+ target_library_key=self.library_v2_1.library_key,
+ composition_level=CompositionLevel.Component,
+ repeat_handling_strategy=RepeatHandlingStrategy.Skip,
preserve_url_slugs=True,
forward_source_to_target=False,
)
@@ -147,11 +206,11 @@ def test_start_migration_to_library_with_strategy_skip(self):
assert modulestoremigration is not None
assert modulestoremigration.repeat_handling_strategy == RepeatHandlingStrategy.Skip.value
- migrated_components_fork = lib_api.get_library_components(self.library_v2.library_key)
+ migrated_components_fork = lib_api.get_library_components(self.library_v2_1.library_key)
assert len(migrated_components_fork) == 1
component = lib_api.LibraryXBlockMetadata.from_component(
- self.library_v2.library_key, migrated_components_fork[0]
+ self.library_v2_1.library_key, migrated_components_fork[0]
)
assert component.display_name == "Original Block"
@@ -173,9 +232,9 @@ def test_start_migration_to_library_with_strategy_update(self):
api.start_migration_to_library(
user=user,
source_key=source.key,
- target_library_key=self.library_v2.library_key,
- composition_level=CompositionLevel.Component.value,
- repeat_handling_strategy=RepeatHandlingStrategy.Skip.value,
+ target_library_key=self.library_v2_1.library_key,
+ composition_level=CompositionLevel.Component,
+ repeat_handling_strategy=RepeatHandlingStrategy.Skip,
preserve_url_slugs=True,
forward_source_to_target=False,
)
@@ -183,7 +242,7 @@ def test_start_migration_to_library_with_strategy_update(self):
modulestoremigration = ModulestoreMigration.objects.get()
assert modulestoremigration.repeat_handling_strategy == RepeatHandlingStrategy.Skip.value
- migrated_components = lib_api.get_library_components(self.library_v2.library_key)
+ migrated_components = lib_api.get_library_components(self.library_v2_1.library_key)
assert len(migrated_components) == 1
# Update the block, changing its name
@@ -194,9 +253,9 @@ def test_start_migration_to_library_with_strategy_update(self):
api.start_migration_to_library(
user=user,
source_key=source.key,
- target_library_key=self.library_v2.library_key,
- composition_level=CompositionLevel.Component.value,
- repeat_handling_strategy=RepeatHandlingStrategy.Update.value,
+ target_library_key=self.library_v2_1.library_key,
+ composition_level=CompositionLevel.Component,
+ repeat_handling_strategy=RepeatHandlingStrategy.Update,
preserve_url_slugs=True,
forward_source_to_target=False,
)
@@ -205,11 +264,11 @@ def test_start_migration_to_library_with_strategy_update(self):
assert modulestoremigration is not None
assert modulestoremigration.repeat_handling_strategy == RepeatHandlingStrategy.Update.value
- migrated_components_fork = lib_api.get_library_components(self.library_v2.library_key)
+ migrated_components_fork = lib_api.get_library_components(self.library_v2_1.library_key)
assert len(migrated_components_fork) == 1
component = lib_api.LibraryXBlockMetadata.from_component(
- self.library_v2.library_key, migrated_components_fork[0]
+ self.library_v2_1.library_key, migrated_components_fork[0]
)
assert component.display_name == "Updated Block"
@@ -231,9 +290,9 @@ def test_start_migration_to_library_with_strategy_forking(self):
api.start_migration_to_library(
user=user,
source_key=source.key,
- target_library_key=self.library_v2.library_key,
- composition_level=CompositionLevel.Component.value,
- repeat_handling_strategy=RepeatHandlingStrategy.Skip.value,
+ target_library_key=self.library_v2_1.library_key,
+ composition_level=CompositionLevel.Component,
+ repeat_handling_strategy=RepeatHandlingStrategy.Skip,
preserve_url_slugs=True,
forward_source_to_target=False,
)
@@ -241,7 +300,7 @@ def test_start_migration_to_library_with_strategy_forking(self):
modulestoremigration = ModulestoreMigration.objects.get()
assert modulestoremigration.repeat_handling_strategy == RepeatHandlingStrategy.Skip.value
- migrated_components = lib_api.get_library_components(self.library_v2.library_key)
+ migrated_components = lib_api.get_library_components(self.library_v2_1.library_key)
assert len(migrated_components) == 1
# Update the block, changing its name
@@ -252,9 +311,9 @@ def test_start_migration_to_library_with_strategy_forking(self):
api.start_migration_to_library(
user=user,
source_key=source.key,
- target_library_key=self.library_v2.library_key,
- composition_level=CompositionLevel.Component.value,
- repeat_handling_strategy=RepeatHandlingStrategy.Fork.value,
+ target_library_key=self.library_v2_1.library_key,
+ composition_level=CompositionLevel.Component,
+ repeat_handling_strategy=RepeatHandlingStrategy.Fork,
preserve_url_slugs=True,
forward_source_to_target=False,
)
@@ -263,16 +322,16 @@ def test_start_migration_to_library_with_strategy_forking(self):
assert modulestoremigration is not None
assert modulestoremigration.repeat_handling_strategy == RepeatHandlingStrategy.Fork.value
- migrated_components_fork = lib_api.get_library_components(self.library_v2.library_key)
+ migrated_components_fork = lib_api.get_library_components(self.library_v2_1.library_key)
assert len(migrated_components_fork) == 2
first_component = lib_api.LibraryXBlockMetadata.from_component(
- self.library_v2.library_key, migrated_components_fork[0]
+ self.library_v2_1.library_key, migrated_components_fork[0]
)
assert first_component.display_name == "Original Block"
second_component = lib_api.LibraryXBlockMetadata.from_component(
- self.library_v2.library_key, migrated_components_fork[1]
+ self.library_v2_1.library_key, migrated_components_fork[1]
)
assert second_component.display_name == "Updated Block"
@@ -284,9 +343,9 @@ def test_start_migration_to_library_with_strategy_forking(self):
api.start_migration_to_library(
user=user,
source_key=source.key,
- target_library_key=self.library_v2.library_key,
- composition_level=CompositionLevel.Component.value,
- repeat_handling_strategy=RepeatHandlingStrategy.Fork.value,
+ target_library_key=self.library_v2_1.library_key,
+ composition_level=CompositionLevel.Component,
+ repeat_handling_strategy=RepeatHandlingStrategy.Fork,
preserve_url_slugs=True,
forward_source_to_target=False,
)
@@ -295,53 +354,215 @@ def test_start_migration_to_library_with_strategy_forking(self):
assert modulestoremigration is not None
assert modulestoremigration.repeat_handling_strategy == RepeatHandlingStrategy.Fork.value
- migrated_components_fork = lib_api.get_library_components(self.library_v2.library_key)
+ migrated_components_fork = lib_api.get_library_components(self.library_v2_1.library_key)
assert len(migrated_components_fork) == 3
first_component = lib_api.LibraryXBlockMetadata.from_component(
- self.library_v2.library_key, migrated_components_fork[0]
+ self.library_v2_1.library_key, migrated_components_fork[0]
)
assert first_component.display_name == "Original Block"
second_component = lib_api.LibraryXBlockMetadata.from_component(
- self.library_v2.library_key, migrated_components_fork[1]
+ self.library_v2_1.library_key, migrated_components_fork[1]
)
assert second_component.display_name == "Updated Block"
third_component = lib_api.LibraryXBlockMetadata.from_component(
- self.library_v2.library_key, migrated_components_fork[2]
+ self.library_v2_1.library_key, migrated_components_fork[2]
)
assert third_component.display_name == "Updated Block Again"
- def test_get_migration_info(self):
+ def test_migration_api_for_various_scenarios(self):
"""
- Test that the API can retrieve migration info.
+ Test that get_migrations, get_block_migrations, forward_context, and forward_block
+ behave as expected throughout a convoluted series of intertwined migrations.
+
+ Also, ensure that each of the aforementioned api functions only performs 1 query each.
"""
+ # pylint: disable=too-many-statements
user = UserFactory()
- collection_key = "test-collection"
+ all_source_usage_keys = {*self.source_html_keys, *self.source_unit_keys}
+ all_source_usage_key_strs = {str(sk) for sk in all_source_usage_keys}
+
+ # In this test, we will be migrating self.lib_v1 a total of 6 times.
+ # We will migrate it to each collection (A, B, and C) twice.
+
+ # Lib 1 has Collection A and Collection B
+ # Lib 2 has Collection C
authoring_api.create_collection(
learning_package_id=self.learning_package.id,
- key=collection_key,
- title="Test Collection",
+ key="test-collection-1a",
+ title="Test Collection A in Lib 1",
+ created_by=user.id,
+ )
+ authoring_api.create_collection(
+ learning_package_id=self.learning_package.id,
+ key="test-collection-1b",
+ title="Test Collection B in Lib 1",
+ created_by=user.id,
+ )
+ authoring_api.create_collection(
+ learning_package_id=self.learning_package_2.id,
+ key="test-collection-2c",
+ title="Test Collection C in Lib 2",
created_by=user.id,
)
+ # No migrations have happened.
+ # Everything should return None / empty.
+ assert not list(api.get_migrations(self.lib_key_v1))
+ assert not api.get_forwarding(source_key=self.lib_key_v1)
+ assert not api.get_forwarding_for_blocks(all_source_usage_keys)
+
+ # FOUR MIGRATIONS!
+ # * Migrate to Lib1.CollA
+ # * Migrate to Lib1.CollB using FORK strategy
+ # * Migrate to Lib1.CollA using UPDATE strategy
+ # * Migrate to Lib2.CollC
+ # Note: None of these are forwarding migrations!
api.start_migration_to_library(
user=user,
- source_key=self.lib_key,
- target_library_key=self.library_v2.library_key,
- target_collection_slug=collection_key,
- composition_level=CompositionLevel.Component.value,
- repeat_handling_strategy=RepeatHandlingStrategy.Skip.value,
+ source_key=self.lib_key_v1,
+ target_library_key=self.lib_key_v2_1,
+ target_collection_slug="test-collection-1a",
+ composition_level=CompositionLevel.Unit,
+ # repeat_handling_strategy here is arbitrary, as there will be no repeats.
+ repeat_handling_strategy=RepeatHandlingStrategy.Skip,
+ preserve_url_slugs=True,
+ forward_source_to_target=False,
+ )
+ api.start_migration_to_library(
+ user=user,
+ source_key=self.lib_key_v1,
+ target_library_key=self.lib_key_v2_1,
+ target_collection_slug="test-collection-1b",
+ composition_level=CompositionLevel.Unit,
+ # this will create a 2nd copy of every block
+ repeat_handling_strategy=RepeatHandlingStrategy.Fork,
+ preserve_url_slugs=True,
+ forward_source_to_target=False,
+ )
+ api.start_migration_to_library(
+ user=user,
+ source_key=self.lib_key_v1,
+ target_library_key=self.lib_key_v2_1,
+ target_collection_slug="test-collection-1a",
+ composition_level=CompositionLevel.Unit,
+ # this will update the 2nd copies, but put them in the same collection as the first copies
+ repeat_handling_strategy=RepeatHandlingStrategy.Update,
+ preserve_url_slugs=True,
+ forward_source_to_target=False,
+ )
+ api.start_migration_to_library(
+ user=user,
+ source_key=self.lib_key_v1,
+ target_library_key=self.lib_key_v2_2,
+ target_collection_slug="test-collection-2c",
+ composition_level=CompositionLevel.Unit,
+ # repeat_handling_strategy here is arbitrary, as there will be no repeats.
+ repeat_handling_strategy=RepeatHandlingStrategy.Skip,
+ preserve_url_slugs=True,
+ forward_source_to_target=False,
+ )
+ # get_migrations returns in reverse chronological order
+ with self.assertNumQueries(1):
+ migration_2c_i, migration_1a_ii, migration_1b_i, migration_1a_i = api.get_migrations(self.lib_key_v1)
+ assert not migration_1a_i.is_failed
+ assert not migration_1b_i.is_failed
+ assert not migration_1a_ii.is_failed
+ assert not migration_2c_i.is_failed
+ # Confirm that the metadata came back correctly.
+ assert migration_1a_i.target_key == self.lib_key_v2_1
+ assert migration_1a_i.target_title == "Test Library 1"
+ assert migration_1a_i.target_collection_slug == "test-collection-1a"
+ assert migration_1a_i.target_collection_title == "Test Collection A in Lib 1"
+ assert migration_2c_i.target_key == self.lib_key_v2_2
+ assert migration_2c_i.target_title == "Test Library 2"
+ assert migration_2c_i.target_collection_slug == "test-collection-2c"
+ assert migration_2c_i.target_collection_title == "Test Collection C in Lib 2"
+ # Call get_migration_blocks on each of the four migrations. Convert the mapping
+ # from UsageKey->BlockMigrationResult into str->str just so we can assert things more easily.
+ with self.assertNumQueries(1):
+ mappings_1a_i = {
+ str(sk): str(bm.target_key) for sk, bm in api.get_migration_blocks(migration_1a_i.pk).items()
+ }
+ mappings_1b_i = {
+ str(sk): str(bm.target_key) for sk, bm in api.get_migration_blocks(migration_1b_i.pk).items()
+ }
+ mappings_1a_ii = {
+ str(sk): str(bm.target_key) for sk, bm in api.get_migration_blocks(migration_1a_ii.pk).items()
+ }
+ mappings_2c_i = {
+ str(sk): str(bm.target_key) for sk, bm in api.get_migration_blocks(migration_2c_i.pk).items()
+ }
+ # Each migration should have migrated every source block.
+ assert set(mappings_1a_i.keys()) == all_source_usage_key_strs
+ assert set(mappings_1b_i.keys()) == all_source_usage_key_strs
+ assert set(mappings_1a_ii.keys()) == all_source_usage_key_strs
+ assert set(mappings_2c_i.keys()) == all_source_usage_key_strs
+ # Because the migration to Lib1.CollB used FORK, we expect that there is nothing in
+ # common between it and the prior migration to Lib1.CollA.
+ assert not (set(mappings_1a_i.values()) & set(mappings_1b_i.values()))
+ # Because the second migration to Lib1.CollA used UPDATE, we expect that it
+ # will have all the same mappings as the prior migration to Lib1.CollB.
+ # This is a little countertuitive, since the migrations targeted different collections,
+ # but the rule that the migrator follows is "UPDATE uses the block from the most recent migration".
+ assert mappings_1b_i == mappings_1a_ii
+ # Since forward_source_to_target=False, we have had no authoritative migration yet.
+ assert api.get_forwarding(self.lib_key_v1) is None
+ assert not api.get_forwarding_for_blocks(all_source_usage_keys)
+
+ # ANOTHER MIGRATION!
+ # * Migrate to Lib2.CollC using UPDATE strategy
+ # Note: This *is* a forwarding migration
+ api.start_migration_to_library(
+ user=user,
+ source_key=self.lib_key_v1,
+ target_library_key=self.lib_key_v2_2,
+ target_collection_slug="test-collection-2c",
+ composition_level=CompositionLevel.Unit,
+ repeat_handling_strategy=RepeatHandlingStrategy.Update,
preserve_url_slugs=True,
forward_source_to_target=True,
)
+ migration_2c_ii, _2c_i, _1a_ii, _1b_i, migration_1a_i_reloaded = api.get_migrations(self.lib_key_v1)
+ assert migration_1a_i_reloaded.pk == migration_1a_i.pk
+ assert not migration_2c_ii.is_failed
+ # Our source lib should now forward to Lib2.
with self.assertNumQueries(1):
- result = api.get_migration_info([self.lib_key])
- row = result.get(self.lib_key)
- assert row is not None
- assert row.migrations__target__key == str(self.lib_key_v2)
- assert row.migrations__target__title == "Test Library"
- assert row.migrations__target_collection__key == collection_key
- assert row.migrations__target_collection__title == "Test Collection"
+ forwarded = api.get_forwarding(self.lib_key_v1)
+ assert forwarded.target_key == self.lib_key_v2_2
+ assert forwarded.target_collection_slug == "test-collection-2c"
+ assert forwarded.pk == migration_2c_ii.pk
+ # Our source lib's blocks should now forward to ones in Lib2.
+ with self.assertNumQueries(1):
+ forwarded_blocks = api.get_forwarding_for_blocks(all_source_usage_keys)
+ assert forwarded_blocks[self.source_html_keys[1]].target_key.context_key == self.lib_key_v2_2
+ assert forwarded_blocks[self.source_unit_keys[1]].target_key.context_key == self.lib_key_v2_2
+
+ # FINAL MIGRATION!
+ # * Migrate to Lib1.CollB using UPDATE strategy
+ # Note: This *is* a forwarding migration, and should supplant the previous
+ # migration for forwarding purposes.
+ api.start_migration_to_library(
+ user=user,
+ source_key=self.lib_key_v1,
+ target_library_key=self.lib_key_v2_1,
+ target_collection_slug="test-collection-1b",
+ composition_level=CompositionLevel.Unit,
+ repeat_handling_strategy=RepeatHandlingStrategy.Update,
+ preserve_url_slugs=True,
+ forward_source_to_target=True,
+ )
+ migration_1b_ii, _2c_ii, _2c_i, _1a_ii, _1b_i, _1a_i = api.get_migrations(self.lib_key_v1)
+ assert not migration_1b_ii.is_failed
+ # Our source lib should now forward to Lib1.
+ forwarded = api.get_forwarding(self.lib_key_v1)
+ assert forwarded.target_key == self.lib_key_v2_1
+ assert forwarded.target_collection_slug == "test-collection-1b"
+ assert forwarded.pk == migration_1b_ii.pk
+ # Our source lib should now forward to Lib1.
+ forwarded_blocks = api.get_forwarding_for_blocks(all_source_usage_keys)
+ assert forwarded_blocks[self.source_html_keys[1]].target_key.context_key == self.lib_key_v2_1
+ assert forwarded_blocks[self.source_unit_keys[1]].target_key.context_key == self.lib_key_v2_1
diff --git a/cms/djangoapps/modulestore_migrator/tests/test_rest_api.py b/cms/djangoapps/modulestore_migrator/tests/test_rest_api.py
new file mode 100644
index 000000000000..4a7f842902db
--- /dev/null
+++ b/cms/djangoapps/modulestore_migrator/tests/test_rest_api.py
@@ -0,0 +1,822 @@
+"""
+Unit tests for the modulestore_migrator REST API v1 views.
+
+These tests focus on validation, HTTP status codes, and serialization/deserialization.
+Business logic is mocked out.
+"""
+from unittest.mock import MagicMock, patch
+from uuid import uuid4
+
+from django.contrib.auth import get_user_model
+from django.test import TestCase
+from opaque_keys.edx.locator import CourseLocator
+from organizations.tests.factories import OrganizationFactory
+from rest_framework import status
+from rest_framework.exceptions import PermissionDenied
+from rest_framework.test import APIRequestFactory, force_authenticate
+from user_tasks.models import UserTaskStatus
+
+from cms.djangoapps.modulestore_migrator.models import (
+ ModulestoreMigration as ModulestoreMigrationModel,
+ ModulestoreSource,
+)
+from cms.djangoapps.modulestore_migrator.rest_api.v1.views import (
+ BulkMigrationViewSet,
+ MigrationViewSet,
+)
+from openedx.core.djangoapps.content_libraries import api as lib_api
+
+
+User = get_user_model()
+
+
+class TestMigrationViewSetCreate(TestCase):
+ """
+ Test the MigrationViewSet.create() endpoint.
+
+ Focus: validation, return codes, serialization/deserialization.
+ """
+
+ def setUp(self):
+ """Set up test fixtures."""
+ self.factory = APIRequestFactory()
+ self.view = MigrationViewSet.as_view({'post': 'create'})
+
+ # Create test user
+ self.user = User.objects.create_user(
+ username='testuser',
+ email='testuser@test.com',
+ password='password'
+ )
+
+ @patch('cms.djangoapps.modulestore_migrator.rest_api.v1.views.migrator_api')
+ @patch('cms.djangoapps.modulestore_migrator.rest_api.v1.views.lib_api')
+ @patch('cms.djangoapps.modulestore_migrator.rest_api.v1.views.auth')
+ @patch('cms.djangoapps.modulestore_migrator.rest_api.v1.views.UserTaskStatus')
+ def test_create_migration_success_with_minimal_data(
+ self, mock_user_task_status, mock_auth, mock_lib_api, mock_migrator_api
+ ):
+ """
+ Test successful migration creation with minimal required fields.
+
+ Validates:
+ - 201 status code is returned
+ - Response contains expected serialized fields
+ - Request data is properly deserialized
+ - Permission checks are performed for both source and target
+ """
+ mock_auth.has_studio_write_access.return_value = True
+ mock_lib_api.require_permission_for_library_key.return_value = None
+
+ mock_task = MagicMock(autospec=True)
+ mock_task.id = 'test-task-id'
+ mock_migrator_api.start_migration_to_library.return_value = mock_task
+
+ mock_task_status = MagicMock(autospec=True)
+ mock_task_status.uuid = uuid4()
+ mock_task_status.state = 'Pending'
+ mock_task_status.state_text = 'Pending'
+ mock_task_status.completed_steps = 0
+ mock_task_status.total_steps = 10
+ mock_task_status.attempts = 1
+ mock_task_status.created = '2025-01-01T00:00:00Z'
+ mock_task_status.modified = '2025-01-01T00:00:00Z'
+ mock_task_status.artifacts = []
+ mock_task_status.migrations.all.return_value = []
+
+ mock_user_task_status.objects.get.return_value = mock_task_status
+
+ request_data = {
+ 'source': 'course-v1:TestOrg+TestCourse+TestRun',
+ 'target': 'lib:TestOrg:TestLibrary',
+ }
+ request = self.factory.post(
+ '/api/modulestore_migrator/v1/migrations/',
+ data=request_data,
+ format='json'
+ )
+ force_authenticate(request, user=self.user)
+
+ response = self.view(request)
+
+ assert response.status_code == status.HTTP_201_CREATED
+
+ assert 'uuid' in response.data
+ assert 'state' in response.data
+ assert 'state_text' in response.data
+ assert 'completed_steps' in response.data
+ assert 'total_steps' in response.data
+ assert 'parameters' in response.data
+
+ mock_auth.has_studio_write_access.assert_called_once()
+ mock_lib_api.require_permission_for_library_key.assert_called_once()
+
+ mock_migrator_api.start_migration_to_library.assert_called_once()
+ call_kwargs = mock_migrator_api.start_migration_to_library.call_args[1]
+ assert call_kwargs['user'] == self.user
+ assert str(call_kwargs['source_key']) == 'course-v1:TestOrg+TestCourse+TestRun'
+ assert str(call_kwargs['target_library_key']) == 'lib:TestOrg:TestLibrary'
+
+ def test_create_migration_invalid_source_key(self):
+ """
+ Test that invalid source key returns 400 Bad Request.
+
+ Validates:
+ - 400 status code is returned
+ - Error message mentions validation failure
+ """
+ request_data = {
+ 'source': 'not-a-valid-key',
+ 'target': 'lib:TestOrg:TestLibrary',
+ }
+ request = self.factory.post(
+ '/api/modulestore_migrator/v1/migrations/',
+ data=request_data,
+ format='json'
+ )
+ force_authenticate(request, user=self.user)
+
+ response = self.view(request)
+
+ assert response.status_code == status.HTTP_400_BAD_REQUEST
+ assert 'source' in response.data
+
+ def test_create_migration_invalid_target_key(self):
+ """
+ Test that invalid target library key returns 400 Bad Request.
+
+ Validates:
+ - 400 status code is returned
+ - Error message mentions target validation failure
+ """
+ request_data = {
+ 'source': 'course-v1:TestOrg+TestCourse+TestRun',
+ 'target': 'not-a-valid-library-key',
+ }
+ request = self.factory.post(
+ '/api/modulestore_migrator/v1/migrations/',
+ data=request_data,
+ format='json'
+ )
+ force_authenticate(request, user=self.user)
+
+ response = self.view(request)
+
+ assert response.status_code == status.HTTP_400_BAD_REQUEST
+ assert 'target' in response.data
+
+ def test_create_migration_missing_required_fields(self):
+ """
+ Test that missing required fields returns 400 Bad Request.
+
+ Validates:
+ - 400 status code is returned when source is missing
+ - 400 status code is returned when target is missing
+ """
+ request_data = {
+ 'target': 'lib:TestOrg:TestLibrary',
+ }
+ request = self.factory.post(
+ '/api/modulestore_migrator/v1/migrations/',
+ data=request_data,
+ format='json'
+ )
+ force_authenticate(request, user=self.user)
+ response = self.view(request)
+
+ assert response.status_code == status.HTTP_400_BAD_REQUEST
+ assert 'source' in response.data
+
+ request_data = {
+ 'source': 'course-v1:TestOrg+TestCourse+TestRun',
+ }
+ request = self.factory.post(
+ '/api/modulestore_migrator/v1/migrations/',
+ data=request_data,
+ format='json'
+ )
+ force_authenticate(request, user=self.user)
+ response = self.view(request)
+
+ assert response.status_code == status.HTTP_400_BAD_REQUEST
+ assert 'target' in response.data
+
+ def test_create_migration_unauthenticated_user(self):
+ """
+ Test that unauthenticated requests return 401 Unauthorized.
+
+ Validates:
+ - 401 status code for unauthenticated requests
+ """
+ request_data = {
+ 'source': 'course-v1:TestOrg+TestCourse+TestRun',
+ 'target': 'lib:TestOrg:TestLibrary',
+ }
+ request = self.factory.post(
+ '/api/modulestore_migrator/v1/migrations/',
+ data=request_data,
+ format='json'
+ )
+ # Note: No force_authenticate call
+
+ response = self.view(request)
+
+ assert response.status_code in [status.HTTP_401_UNAUTHORIZED, status.HTTP_403_FORBIDDEN]
+
+ @patch('cms.djangoapps.modulestore_migrator.rest_api.v1.views.auth')
+ def test_create_migration_without_source_author_access(self, mock_auth):
+ """
+ Test that users without author access to source cannot create migrations.
+
+ Validates:
+ - 403 Forbidden status code when user lacks author access to source
+ """
+ mock_auth.has_studio_write_access.return_value = False
+
+ request_data = {
+ 'source': 'course-v1:TestOrg+TestCourse+TestRun',
+ 'target': 'lib:TestOrg:TestLibrary',
+ }
+ request = self.factory.post(
+ '/api/modulestore_migrator/v1/migrations/',
+ data=request_data,
+ format='json'
+ )
+ force_authenticate(request, user=self.user)
+
+ response = self.view(request)
+
+ assert response.status_code == status.HTTP_403_FORBIDDEN
+
+ @patch('cms.djangoapps.modulestore_migrator.rest_api.v1.views.lib_api')
+ @patch('cms.djangoapps.modulestore_migrator.rest_api.v1.views.auth')
+ def test_create_migration_without_target_write_access(self, mock_auth, mock_lib_api):
+ """
+ Test that users without write access to target cannot create migrations.
+
+ Validates:
+ - 403 Forbidden status code when user lacks write access to target library
+ """
+ mock_auth.has_studio_write_access.return_value = True
+ mock_lib_api.require_permission_for_library_key.side_effect = PermissionDenied(
+ "User lacks permission to manage content in this library"
+ )
+
+ request_data = {
+ 'source': 'course-v1:TestOrg+TestCourse+TestRun',
+ 'target': 'lib:TestOrg:TestLibrary',
+ }
+ request = self.factory.post(
+ '/api/modulestore_migrator/v1/migrations/',
+ data=request_data,
+ format='json'
+ )
+ force_authenticate(request, user=self.user)
+
+ response = self.view(request)
+
+ assert response.status_code == status.HTTP_403_FORBIDDEN
+
+ @patch('cms.djangoapps.modulestore_migrator.rest_api.v1.views.migrator_api')
+ @patch('cms.djangoapps.modulestore_migrator.rest_api.v1.views.lib_api')
+ @patch('cms.djangoapps.modulestore_migrator.rest_api.v1.views.auth')
+ @patch('cms.djangoapps.modulestore_migrator.rest_api.v1.views.UserTaskStatus')
+ def test_create_migration_with_optional_fields(
+ self, mock_user_task_status, mock_auth, mock_lib_api, mock_migrator_api
+ ):
+ """
+ Test migration creation with all optional fields provided.
+
+ Validates:
+ - Optional fields are properly deserialized
+ - Default values are not used when explicit values provided
+ """
+ mock_auth.has_studio_write_access.return_value = True
+ mock_lib_api.require_permission_for_library_key.return_value = None
+
+ mock_task = MagicMock(autospec=True)
+ mock_task.id = 'test-task-id'
+ mock_migrator_api.start_migration_to_library.return_value = mock_task
+
+ mock_task_status = MagicMock(autospec=True)
+ mock_task_status.uuid = uuid4()
+ mock_task_status.state = 'Pending'
+ mock_task_status.state_text = 'Pending'
+ mock_task_status.completed_steps = 0
+ mock_task_status.total_steps = 10
+ mock_task_status.attempts = 1
+ mock_task_status.created = '2025-01-01T00:00:00Z'
+ mock_task_status.modified = '2025-01-01T00:00:00Z'
+ mock_task_status.artifacts = []
+ mock_task_status.migrations.all.return_value = []
+
+ mock_user_task_status.objects.get.return_value = mock_task_status
+
+ request_data = {
+ 'source': 'course-v1:TestOrg+TestCourse+TestRun',
+ 'target': 'lib:TestOrg:TestLibrary',
+ 'target_collection_slug': 'my-collection',
+ 'composition_level': 'unit',
+ 'repeat_handling_strategy': 'update',
+ 'preserve_url_slugs': False,
+ 'forward_source_to_target': True,
+ }
+ request = self.factory.post(
+ '/api/modulestore_migrator/v1/migrations/',
+ data=request_data,
+ format='json'
+ )
+ force_authenticate(request, user=self.user)
+
+ response = self.view(request)
+
+ assert response.status_code == status.HTTP_201_CREATED
+
+ mock_migrator_api.start_migration_to_library.assert_called_once()
+ call_kwargs = mock_migrator_api.start_migration_to_library.call_args[1]
+ assert call_kwargs['target_collection_slug'] == 'my-collection'
+ # CompositionLevel and RepeatHandlingStrategy are enums
+ assert call_kwargs['composition_level'].value == 'unit'
+ assert call_kwargs['repeat_handling_strategy'].value == 'update'
+ assert call_kwargs['preserve_url_slugs'] is False
+ assert call_kwargs['forward_source_to_target'] is True
+
+ def test_create_migration_invalid_composition_level(self):
+ """
+ Test that invalid composition_level returns 400 Bad Request.
+
+ Validates:
+ - 400 status code for invalid enum value
+ """
+ request_data = {
+ 'source': 'course-v1:TestOrg+TestCourse+TestRun',
+ 'target': 'lib:TestOrg:TestLibrary',
+ 'composition_level': 'invalid_level',
+ }
+ request = self.factory.post(
+ '/api/modulestore_migrator/v1/migrations/',
+ data=request_data,
+ format='json'
+ )
+ force_authenticate(request, user=self.user)
+
+ response = self.view(request)
+
+ assert response.status_code == status.HTTP_400_BAD_REQUEST
+ assert 'composition_level' in response.data
+
+ def test_create_migration_invalid_repeat_handling_strategy(self):
+ """
+ Test that invalid repeat_handling_strategy returns 400 Bad Request.
+
+ Validates:
+ - 400 status code for invalid enum value
+ """
+ request_data = {
+ 'source': 'course-v1:TestOrg+TestCourse+TestRun',
+ 'target': 'lib:TestOrg:TestLibrary',
+ 'repeat_handling_strategy': 'invalid_strategy',
+ }
+ request = self.factory.post(
+ '/api/modulestore_migrator/v1/migrations/',
+ data=request_data,
+ format='json'
+ )
+ force_authenticate(request, user=self.user)
+
+ response = self.view(request)
+
+ assert response.status_code == status.HTTP_400_BAD_REQUEST
+ assert 'repeat_handling_strategy' in response.data
+
+
+class TestMigrationViewSetList(TestCase):
+ """
+ Test the MigrationViewSet.list() endpoint.
+
+ Focus: validation, return codes, serialization/deserialization.
+ """
+
+ def setUp(self):
+ """Set up test fixtures."""
+ self.factory = APIRequestFactory()
+ self.view = MigrationViewSet.as_view({'get': 'list'})
+
+ self.user = User.objects.create_user(
+ username='testuser',
+ email='testuser@test.com',
+ password='password'
+ )
+ self.other_user = User.objects.create_user(
+ username='otheruser',
+ email='otheruser@test.com',
+ password='password'
+ )
+
+ def test_list_migrations_success(self):
+ """
+ Test successful listing of migrations for the authenticated user.
+
+ Validates:
+ - 200 status code is returned
+ - Response contains list of migrations
+ - Only user's own migrations are returned (other users' migrations filtered out)
+ """
+ org = OrganizationFactory(short_name="TestOrg", name="Test Org")
+ source_key = CourseLocator.from_string('course-v1:TestOrg+TestCourse+TestRun')
+ source = ModulestoreSource.objects.create(key=str(source_key))
+ target = lib_api.create_library(org=org, slug="TestLib", title="Test Target Lib")
+ user_task_status = UserTaskStatus.objects.create(
+ user=self.user,
+ task_id='user-task-id',
+ task_class='test.Task',
+ name='User Migration',
+ total_steps=10,
+ completed_steps=10,
+ )
+ other_task_status = UserTaskStatus.objects.create(
+ user=self.other_user,
+ task_id='other-task-id',
+ task_class='test.Task',
+ name='Other Migration',
+ total_steps=5,
+ completed_steps=5,
+ )
+ ModulestoreMigrationModel.objects.create(
+ task_status=user_task_status,
+ source=source,
+ target_id=target.learning_package_id,
+ )
+ ModulestoreMigrationModel.objects.create(
+ task_status=other_task_status,
+ source=source,
+ target_id=target.learning_package_id,
+ )
+
+ request = self.factory.get('/api/modulestore_migrator/v1/migrations/')
+ force_authenticate(request, user=self.user)
+ response = self.view(request)
+
+ assert response.status_code == status.HTTP_200_OK
+ results = response.data['results']
+ assert len(results) == 1
+ assert results[0]['uuid'] == str(user_task_status.uuid)
+
+ def test_list_migrations_unauthenticated(self):
+ """
+ Test that unauthenticated requests return 401 Unauthorized.
+
+ Validates:
+ - 401 status code for unauthenticated requests
+ """
+ request = self.factory.get('/api/modulestore_migrator/v1/migrations/')
+
+ response = self.view(request)
+
+ assert response.status_code == status.HTTP_401_UNAUTHORIZED
+
+
+class TestMigrationViewSetRetrieve(TestCase):
+ """
+ Test the MigrationViewSet.retrieve() endpoint.
+
+ Focus: validation, return codes, serialization/deserialization.
+ """
+
+ def setUp(self):
+ """Set up test fixtures."""
+ self.factory = APIRequestFactory()
+ self.view = MigrationViewSet.as_view({'get': 'retrieve'})
+
+ self.user = User.objects.create_user(
+ username='testuser',
+ email='testuser@test.com',
+ password='password'
+ )
+
+ def test_retrieve_migration_success(self):
+ """
+ Test successful retrieval of a specific migration by UUID.
+
+ Validates:
+ - 200 status code is returned
+ - Response contains migration details
+ """
+ org = OrganizationFactory(short_name="TestOrg", name="Test Org")
+ source_key = CourseLocator.from_string('course-v1:TestOrg+TestCourse+TestRun')
+ source = ModulestoreSource.objects.create(key=str(source_key))
+ target = lib_api.create_library(org=org, slug="TestLib", title="Test Target Lib")
+ user_task_status = UserTaskStatus.objects.create(
+ user=self.user,
+ task_id='user-task-id',
+ task_class='test.Task',
+ name='User Migration',
+ total_steps=10,
+ completed_steps=10,
+ )
+ ModulestoreMigrationModel.objects.create(
+ task_status=user_task_status,
+ source=source,
+ target_id=target.learning_package_id,
+ )
+
+ request = self.factory.get(f'/api/modulestore_migrator/v1/migrations/{user_task_status.uuid}/')
+ force_authenticate(request, user=self.user)
+ response = self.view(request, uuid=str(user_task_status.uuid))
+
+ assert response.status_code == status.HTTP_200_OK
+ assert response.data['uuid'] == str(user_task_status.uuid)
+ assert 'parameters' in response.data
+
+ def test_retrieve_migration_other_user(self):
+ """
+ Test that users cannot retrieve migrations created by other users.
+
+ Validates:
+ - 404 status code when attempting to retrieve another user's migration
+ - Users are isolated to their own migrations
+ """
+ other_user = User.objects.create_user(
+ username='otheruser',
+ email='other@test.com',
+ password='password'
+ )
+ org = OrganizationFactory(short_name="TestOrg", name="Test Org")
+ source_key = CourseLocator.from_string('course-v1:TestOrg+TestCourse+TestRun')
+ source = ModulestoreSource.objects.create(key=str(source_key))
+ target = lib_api.create_library(org=org, slug="TestLib", title="Test Target Lib")
+ other_task_status = UserTaskStatus.objects.create(
+ user=other_user,
+ task_id='other-task-id',
+ task_class='test.Task',
+ name='Other Migration',
+ total_steps=10,
+ completed_steps=10,
+ )
+ ModulestoreMigrationModel.objects.create(
+ task_status=other_task_status,
+ source=source,
+ target_id=target.learning_package_id,
+ )
+
+ request = self.factory.get(f'/api/modulestore_migrator/v1/migrations/{other_task_status.uuid}/')
+ force_authenticate(request, user=self.user)
+ response = self.view(request, uuid=str(other_task_status.uuid))
+
+ assert response.status_code == status.HTTP_404_NOT_FOUND
+
+ def test_retrieve_migration_unauthenticated(self):
+ """
+ Test that unauthenticated requests return 401 Unauthorized.
+
+ Validates:
+ - 401 status code for unauthenticated requests
+ """
+ task_uuid = uuid4()
+ request = self.factory.get(f'/api/modulestore_migrator/v1/migrations/{task_uuid}/')
+
+ response = self.view(request, uuid=str(task_uuid))
+
+ assert response.status_code == status.HTTP_401_UNAUTHORIZED
+
+
+class TestMigrationViewSetCancel(TestCase):
+ """
+ Test the MigrationViewSet.cancel() endpoint.
+
+ Focus: validation, return codes, authorization.
+ """
+
+ def setUp(self):
+ """Set up test fixtures."""
+ self.factory = APIRequestFactory()
+ self.view = MigrationViewSet.as_view({'post': 'cancel'})
+
+ self.staff_user = User.objects.create_user(
+ username='staffuser',
+ email='staff@test.com',
+ password='password',
+ is_staff=True
+ )
+ self.regular_user = User.objects.create_user(
+ username='regularuser',
+ email='regular@test.com',
+ password='password',
+ is_staff=False
+ )
+
+ def test_cancel_migration_as_staff(self):
+ """
+ Test that staff users can cancel migrations.
+
+ Validates:
+ - Staff users can successfully cancel migrations
+ - UserTaskStatus.cancel is called
+ """
+ org = OrganizationFactory(short_name="TestOrg", name="Test Org")
+ source_key = CourseLocator.from_string('course-v1:TestOrg+TestCourse+TestRun')
+ source = ModulestoreSource.objects.create(key=str(source_key))
+ target = lib_api.create_library(org=org, slug="TestLib", title="Test Target Lib")
+ user_task_status = UserTaskStatus.objects.create(
+ user=self.staff_user,
+ task_id='staff-task-id',
+ task_class='test.Task',
+ name='Staff Migration',
+ total_steps=10,
+ completed_steps=5,
+ )
+ ModulestoreMigrationModel.objects.create(
+ task_status=user_task_status,
+ source=source,
+ target_id=target.learning_package_id,
+ )
+
+ with patch.object(UserTaskStatus, 'cancel') as mock_cancel:
+ request = self.factory.post(
+ f'/api/modulestore_migrator/v1/migrations/{user_task_status.uuid}/cancel/'
+ )
+ force_authenticate(request, user=self.staff_user)
+ response = self.view(request, uuid=str(user_task_status.uuid))
+
+ assert response.status_code == status.HTTP_200_OK
+ mock_cancel.assert_called_once()
+
+ def test_cancel_migration_not_found(self):
+ """
+ Test that attempting to cancel a non-existent migration returns 404.
+
+ Validates:
+ - 404 status code when migration UUID does not exist
+ """
+ nonexistent_uuid = uuid4()
+ request = self.factory.post(
+ f'/api/modulestore_migrator/v1/migrations/{nonexistent_uuid}/cancel/'
+ )
+ force_authenticate(request, user=self.staff_user)
+
+ response = self.view(request, uuid=str(nonexistent_uuid))
+
+ assert response.status_code == status.HTTP_404_NOT_FOUND
+
+ def test_cancel_migration_as_non_staff(self):
+ """
+ Test that non-staff users cannot cancel migrations.
+
+ Validates:
+ - 403 Forbidden status code for non-staff users
+ """
+ task_uuid = uuid4()
+ request = self.factory.post(
+ f'/api/modulestore_migrator/v1/migrations/{task_uuid}/cancel/'
+ )
+ force_authenticate(request, user=self.regular_user)
+
+ response = self.view(request, uuid=str(task_uuid))
+
+ assert response.status_code == status.HTTP_403_FORBIDDEN
+
+ def test_cancel_migration_unauthenticated(self):
+ """
+ Test that unauthenticated users cannot cancel migrations.
+ """
+ task_uuid = uuid4()
+ request = self.factory.post(
+ f'/api/modulestore_migrator/v1/migrations/{task_uuid}/cancel/'
+ )
+
+ response = self.view(request, uuid=str(task_uuid))
+
+ assert response.status_code == status.HTTP_401_UNAUTHORIZED
+
+
+class TestBulkMigrationViewSetCreate(TestCase):
+ """
+ Test the BulkMigrationViewSet.create() endpoint.
+
+ Focus: validation, return codes, serialization/deserialization.
+ """
+
+ def setUp(self):
+ """Set up test fixtures."""
+ self.factory = APIRequestFactory()
+ self.view = BulkMigrationViewSet.as_view({'post': 'create'})
+
+ self.user = User.objects.create_user(
+ username='testuser',
+ email='testuser@test.com',
+ password='password'
+ )
+
+ @patch('cms.djangoapps.modulestore_migrator.rest_api.v1.views.migrator_api')
+ @patch('cms.djangoapps.modulestore_migrator.rest_api.v1.views.lib_api')
+ @patch('cms.djangoapps.modulestore_migrator.rest_api.v1.views.auth')
+ @patch('cms.djangoapps.modulestore_migrator.rest_api.v1.views.UserTaskStatus')
+ def test_create_bulk_migration_success(
+ self, mock_user_task_status, mock_auth, mock_lib_api, mock_migrator_api
+ ):
+ """
+ Test successful bulk migration creation with multiple sources.
+
+ Validates:
+ - 201 status code is returned
+ - Response contains expected serialized fields
+ - Multiple sources are properly deserialized
+ """
+ mock_auth.has_studio_write_access.return_value = True
+ mock_lib_api.require_permission_for_library_key.return_value = None
+
+ mock_task = MagicMock(autospec=True)
+ mock_task.id = 'test-task-id'
+ mock_migrator_api.start_bulk_migration_to_library.return_value = mock_task
+
+ mock_task_status = MagicMock(autospec=True)
+ mock_task_status.uuid = uuid4()
+ mock_task_status.state = 'Pending'
+ mock_task_status.state_text = 'Pending'
+ mock_task_status.completed_steps = 0
+ mock_task_status.total_steps = 10
+ mock_task_status.attempts = 1
+ mock_task_status.created = '2025-01-01T00:00:00Z'
+ mock_task_status.modified = '2025-01-01T00:00:00Z'
+ mock_task_status.artifacts = []
+ mock_task_status.migrations.all.return_value = []
+
+ mock_user_task_status.objects.get.return_value = mock_task_status
+
+ request_data = {
+ 'sources': [
+ 'course-v1:TestOrg+TestCourse1+Run1',
+ 'course-v1:TestOrg+TestCourse2+Run2'
+ ],
+ 'target': 'lib:TestOrg:TestLibrary',
+ }
+ request = self.factory.post(
+ '/api/modulestore_migrator/v1/bulk_migration/',
+ data=request_data,
+ format='json'
+ )
+ force_authenticate(request, user=self.user)
+
+ response = self.view(request)
+
+ assert response.status_code == status.HTTP_201_CREATED
+ assert 'uuid' in response.data
+ assert 'parameters' in response.data
+
+ mock_migrator_api.start_bulk_migration_to_library.assert_called_once()
+ call_kwargs = mock_migrator_api.start_bulk_migration_to_library.call_args[1]
+ assert call_kwargs['source_key_list'] == [
+ CourseLocator.from_string('course-v1:TestOrg+TestCourse1+Run1'),
+ CourseLocator.from_string('course-v1:TestOrg+TestCourse2+Run2'),
+ ]
+ assert call_kwargs['target_collection_slug_list'] is None
+ assert call_kwargs['create_collections'] is False
+ # CompositionLevel and RepeatHandlingStrategy are enums
+ assert call_kwargs['composition_level'].value == 'component'
+ assert call_kwargs['repeat_handling_strategy'].value == 'skip'
+ assert call_kwargs['preserve_url_slugs'] is False
+ assert call_kwargs['forward_source_to_target'] is None
+
+ def test_create_bulk_migration_invalid_source_key(self):
+ """
+ Test that invalid source key in list returns 400 Bad Request.
+
+ Validates:
+ - 400 status code when one or more sources are invalid
+ """
+ request_data = {
+ 'sources': ['not-a-valid-key', 'course-v1:TestOrg+TestCourse+TestRun'],
+ 'target': 'lib:TestOrg:TestLibrary',
+ }
+ request = self.factory.post(
+ '/api/modulestore_migrator/v1/bulk_migration/',
+ data=request_data,
+ format='json'
+ )
+ force_authenticate(request, user=self.user)
+
+ response = self.view(request)
+
+ assert response.status_code == status.HTTP_400_BAD_REQUEST
+ assert 'sources' in response.data
+
+ def test_create_bulk_migration_missing_sources(self):
+ """
+ Test that missing sources field returns 400 Bad Request.
+
+ Validates:
+ - 400 status code when sources is missing
+ """
+ request_data = {
+ 'target': 'lib:TestOrg:TestLibrary',
+ }
+ request = self.factory.post(
+ '/api/modulestore_migrator/v1/bulk_migration/',
+ data=request_data,
+ format='json'
+ )
+ force_authenticate(request, user=self.user)
+ response = self.view(request)
+
+ assert response.status_code == status.HTTP_400_BAD_REQUEST
+ assert 'sources' in response.data
diff --git a/cms/djangoapps/modulestore_migrator/tests/test_tasks.py b/cms/djangoapps/modulestore_migrator/tests/test_tasks.py
index 309877ae0d0b..02ac7c6f2bbc 100644
--- a/cms/djangoapps/modulestore_migrator/tests/test_tasks.py
+++ b/cms/djangoapps/modulestore_migrator/tests/test_tasks.py
@@ -2,43 +2,46 @@
Tests for the modulestore_migrator tasks
"""
-from unittest.mock import Mock
+from unittest.mock import Mock, patch
+
import ddt
from django.utils import timezone
from lxml import etree
from opaque_keys.edx.keys import CourseKey
from opaque_keys.edx.locator import LibraryLocator, LibraryLocatorV2
-from openedx_learning.api.authoring_models import Collection, PublishableEntityVersion
from openedx_learning.api import authoring as authoring_api
+from openedx_learning.api.authoring_models import Collection, PublishableEntityVersion
from organizations.tests.factories import OrganizationFactory
from user_tasks.models import UserTaskArtifact
from user_tasks.tasks import UserTaskStatus
-from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase
-from xmodule.modulestore.tests.factories import CourseFactory
-from common.djangoapps.student.tests.factories import UserFactory
from cms.djangoapps.modulestore_migrator.data import CompositionLevel, RepeatHandlingStrategy
from cms.djangoapps.modulestore_migrator.models import (
ModulestoreMigration,
ModulestoreSource,
)
from cms.djangoapps.modulestore_migrator.tasks import (
+ MigrationStep,
+ _BulkMigrationTask,
_migrate_component,
_migrate_container,
_migrate_node,
_MigratedNode,
_MigrationContext,
- _MigrationTask,
- migrate_from_modulestore,
- MigrationStep,
+ bulk_migrate_from_modulestore,
)
+from common.djangoapps.student.tests.factories import UserFactory
from openedx.core.djangoapps.content_libraries import api as lib_api
+from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase
+from xmodule.modulestore.tests.factories import CourseFactory, LibraryFactory, BlockFactory
+
+from .. import api as migrator_api
@ddt.ddt
class TestMigrateFromModulestore(ModuleStoreTestCase):
"""
- Test the migrate_from_modulestore task
+ Test the bulk_migrate_from_modulestore task
"""
def setUp(self):
@@ -48,24 +51,77 @@ def setUp(self):
self.lib_key = LibraryLocatorV2.from_string(
f"lib:{self.organization.short_name}:test-key"
)
+ self.lib_key_2 = LibraryLocatorV2.from_string(
+ f"lib:{self.organization.short_name}:test-key-2"
+ )
lib_api.create_library(
org=self.organization,
slug=self.lib_key.slug,
title="Test Library",
)
+ lib_api.create_library(
+ org=self.organization,
+ slug=self.lib_key_2.slug,
+ title="Test Library 2",
+ )
self.library = lib_api.ContentLibrary.objects.get(slug=self.lib_key.slug)
+ self.library_2 = lib_api.ContentLibrary.objects.get(slug=self.lib_key_2.slug)
self.learning_package = self.library.learning_package
+ self.learning_package_2 = self.library_2.learning_package
self.course = CourseFactory(
org=self.organization.short_name,
course="TestCourse",
run="TestRun",
display_name="Test Course",
)
+ self.course_2 = CourseFactory(
+ org=self.organization.short_name,
+ course="TestCourse2",
+ run="TestRun2",
+ display_name="Test Course 2",
+ )
+ self.legacy_library = LibraryFactory(
+ org=self.organization.short_name,
+ library="LegacyLibrary",
+ display_name="Legacy Library",
+ )
+ self.legacy_library_2 = LibraryFactory(
+ org=self.organization.short_name,
+ library="LegacyLibrary2",
+ display_name="Legacy Library 2",
+ )
self.collection = Collection.objects.create(
learning_package=self.learning_package,
key="test_collection",
title="Test Collection",
)
+ self.collection2 = Collection.objects.create(
+ learning_package=self.learning_package,
+ key="test_collection2",
+ title="Test Collection 2",
+ )
+
+ def _make_migration_context(self, **kwargs) -> _MigrationContext:
+ """
+ Builds a _MigrationContext object with default values, overridable with kwargs
+ """
+ return _MigrationContext(
+ **{
+ "used_component_keys": set(),
+ "used_container_slugs": set(),
+ "previous_block_migrations": {},
+ "target_package_id": self.learning_package.id,
+ "target_library_key": self.library.library_key,
+ "source_context_key": self.course.id,
+ "content_by_filename": {},
+ "composition_level": CompositionLevel.Unit,
+ "repeat_handling_strategy": RepeatHandlingStrategy.Skip,
+ "preserve_url_slugs": True,
+ "created_at": timezone.now(),
+ "created_by": self.user.id,
+ **kwargs,
+ },
+ )
def _get_task_status_fail_message(self, status):
"""
@@ -80,18 +136,7 @@ def test_migrate_node_wiki_tag(self):
Test _migrate_node ignores wiki tags
"""
wiki_node = etree.fromstring("
+
+ ''')
+ context = self._make_migration_context()
result = _migrate_node(
context=context,
source_node=node_without_url_name,
)
- self.assertIsNone(result.source_to_target)
+ self.assertEqual(
+ result.source_to_target,
+ (
+ self.course.id.make_usage_key('library_content', 'test_library_content'),
+ None,
+ 'The "library_content" XBlock (ID: "test_library_content") has children, '
+ 'so it not supported in content libraries. It has 2 children blocks.',
+ ),
+ )
self.assertEqual(len(result.children), 0)
def test_migrated_node_all_source_to_target_pairs(self):
@@ -248,11 +274,11 @@ def test_migrated_node_all_source_to_target_pairs(self):
key2 = self.course.id.make_usage_key("problem", "problem2")
key3 = self.course.id.make_usage_key("problem", "problem3")
- child_node = _MigratedNode(source_to_target=(key3, mock_version3), children=[])
+ child_node = _MigratedNode(source_to_target=(key3, mock_version3, None), children=[])
parent_node = _MigratedNode(
- source_to_target=(key1, mock_version1),
+ source_to_target=(key1, mock_version1, None),
children=[
- _MigratedNode(source_to_target=(key2, mock_version2), children=[]),
+ _MigratedNode(source_to_target=(key2, mock_version2, None), children=[]),
child_node,
],
)
@@ -264,17 +290,16 @@ def test_migrated_node_all_source_to_target_pairs(self):
self.assertEqual(pairs[1][0], key2)
self.assertEqual(pairs[2][0], key3)
- def test_migrate_from_modulestore_invalid_source(self):
+ def test_bulk_migrate_invalid_sources(self):
"""
- Test migrate_from_modulestore with invalid source
+ Test bulk_migrate_from_modulestore with invalid source
"""
- task = migrate_from_modulestore.apply_async(
+ task = bulk_migrate_from_modulestore.apply_async(
kwargs={
"user_id": self.user.id,
- "source_pk": 999999, # Non-existent source
- "target_package_pk": self.learning_package.id,
+ "sources_pks": [999999], # Non-existent source
"target_library_key": str(self.lib_key),
- "target_collection_pk": self.collection.id,
+ "target_collection_pks": [self.collection.id],
"repeat_handling_strategy": RepeatHandlingStrategy.Skip.value,
"preserve_url_slugs": True,
"composition_level": CompositionLevel.Unit.value,
@@ -286,47 +311,20 @@ def test_migrate_from_modulestore_invalid_source(self):
self.assertEqual(status.state, UserTaskStatus.FAILED)
self.assertEqual(self._get_task_status_fail_message(status), "ModulestoreSource matching query does not exist.")
- def test_migrate_from_modulestore_invalid_target_package(self):
- """
- Test migrate_from_modulestore with invalid target package
- """
- source = ModulestoreSource.objects.create(
- key=self.course.id,
- )
-
- task = migrate_from_modulestore.apply_async(
- kwargs={
- "user_id": self.user.id,
- "source_pk": source.id,
- "target_package_pk": 999999, # Non-existent package
- "target_library_key": str(self.lib_key),
- "target_collection_pk": self.collection.id,
- "repeat_handling_strategy": RepeatHandlingStrategy.Skip.value,
- "preserve_url_slugs": True,
- "composition_level": CompositionLevel.Unit.value,
- "forward_source_to_target": False,
- }
- )
-
- status = UserTaskStatus.objects.get(task_id=task.id)
- self.assertEqual(status.state, UserTaskStatus.FAILED)
- self.assertEqual(self._get_task_status_fail_message(status), "LearningPackage matching query does not exist.")
-
- def test_migrate_from_modulestore_invalid_collection(self):
+ def test_bulk_migrate_invalid_collection(self):
"""
- Test migrate_from_modulestore with invalid collection
+ Test bulk_migrate_from_modulestore with invalid collection
"""
source = ModulestoreSource.objects.create(
key=self.course.id,
)
- task = migrate_from_modulestore.apply_async(
+ task = bulk_migrate_from_modulestore.apply_async(
kwargs={
"user_id": self.user.id,
- "source_pk": source.id,
- "target_package_pk": self.learning_package.id,
+ "sources_pks": [source.id],
"target_library_key": str(self.lib_key),
- "target_collection_pk": 999999, # Non-existent collection
+ "target_collection_pks": [999999], # Non-existent collection
"repeat_handling_strategy": RepeatHandlingStrategy.Skip.value,
"preserve_url_slugs": True,
"composition_level": CompositionLevel.Unit.value,
@@ -338,12 +336,14 @@ def test_migrate_from_modulestore_invalid_collection(self):
self.assertEqual(status.state, UserTaskStatus.FAILED)
self.assertEqual(self._get_task_status_fail_message(status), "Collection matching query does not exist.")
- def test_migration_task_calculate_total_steps(self):
+ def test_bulk_migration_task_calculate_total_steps(self):
"""
- Test _MigrationTask.calculate_total_steps returns correct count
+ Test _BulkMigrationTask.calculate_total_steps returns correct count
"""
- total_steps = _MigrationTask.calculate_total_steps({})
- expected_steps = len(list(MigrationStep))
+ total_steps = _BulkMigrationTask.calculate_total_steps({
+ "sources_pks": [1, 2, 3, 4],
+ })
+ expected_steps = len(list(MigrationStep)) - 1 + 6 * 3
self.assertEqual(total_steps, expected_steps)
def test_migrate_component_success(self):
@@ -352,26 +352,15 @@ def test_migrate_component_success(self):
"""
source_key = self.course.id.make_usage_key("problem", "test_problem")
olx = '
+
+ '''
+ context = self._make_migration_context()
+ result, reason = _migrate_component(
+ context=context,
+ source_key=source_key,
+ olx=olx,
+ title="test_library content"
+ )
+
+ self.assertIsNone(result)
+ self.assertEqual(
+ reason,
+ 'The "library_content" XBlock (ID: "test_library_content") has children,'
+ ' so it not supported in content libraries.',
+ )
+
def test_migrate_component_with_static_content(self):
"""
Test _migrate_component with static file content
@@ -394,19 +412,8 @@ def test_migrate_component_with_static_content(self):
created=timezone.now(),
)
content_by_filename = {"test_image.png": test_content.id}
- context = _MigrationContext(
- existing_source_to_target_keys={},
- target_package_id=self.learning_package.id,
- target_library_key=self.library.library_key,
- source_context_key=self.course.id,
- content_by_filename=content_by_filename,
- composition_level=CompositionLevel.Unit,
- repeat_handling_strategy=RepeatHandlingStrategy.Skip,
- preserve_url_slugs=True,
- created_at=timezone.now(),
- created_by=self.user.id,
- )
- result = _migrate_component(
+ context = self._make_migration_context(content_by_filename=content_by_filename)
+ result, reason = _migrate_component(
context=context,
source_key=source_key,
olx=olx,
@@ -414,6 +421,7 @@ def test_migrate_component_with_static_content(self):
)
self.assertIsNotNone(result)
+ self.assertIsNone(reason)
component_content = result.componentversion.componentversioncontent_set.filter(
key="static/test_image.png"
@@ -421,208 +429,245 @@ def test_migrate_component_with_static_content(self):
self.assertIsNotNone(component_content)
self.assertEqual(component_content.content_id, test_content.id)
- def test_migrate_component_replace_existing_false(self):
+ def test_migrate_skip_repeats(self):
"""
- Test _migrate_component with replace_existing=False returns existing component
+ Test that, when requested, the migration will Skip blocks that have previously been migrated
+
+ Tests with both a container and a component
"""
- source_key = self.course.id.make_usage_key("problem", "existing_problem")
- olx = ')
('integrated_channels.integrated_channel', None),
('integrated_channels.degreed', None),
('integrated_channels.degreed2', None),
@@ -1425,8 +1422,6 @@
XBLOCK_FS_STORAGE_BUCKET = None
XBLOCK_FS_STORAGE_PREFIX = None
-STUDIO_FRONTEND_CONTAINER_URL = None
-
############################ Global Database Configuration #####################
DATABASE_ROUTERS = [
diff --git a/cms/lib/xblock/upstream_sync.py b/cms/lib/xblock/upstream_sync.py
index da9a60b422bc..fc79922cc348 100644
--- a/cms/lib/xblock/upstream_sync.py
+++ b/cms/lib/xblock/upstream_sync.py
@@ -17,17 +17,17 @@
import logging
import typing as t
-from dataclasses import dataclass, asdict
+from dataclasses import asdict, dataclass
from django.conf import settings
from django.utils.translation import gettext_lazy as _
from opaque_keys import InvalidKeyError
-from opaque_keys.edx.keys import CourseKey
+from opaque_keys.edx.keys import CourseKey, UsageKey
from opaque_keys.edx.locator import LibraryContainerLocator, LibraryUsageLocatorV2
-from opaque_keys.edx.keys import UsageKey
+from xblock.core import XBlock, XBlockMixin
from xblock.exceptions import XBlockNotFoundError
-from xblock.fields import Scope, String, Integer, Dict, List
-from xblock.core import XBlockMixin, XBlock
+from xblock.fields import Integer, List, Scope, String
+
from xmodule.util.keys import BlockKey
if t.TYPE_CHECKING:
@@ -37,6 +37,13 @@
logger = logging.getLogger(__name__)
+class _UpstreamKeyString(String):
+ """
+ A String field variant that never serializes a None value as an XML child element.
+ """
+ none_to_xml = False
+
+
class UpstreamLinkException(Exception):
"""
Raised whenever we try to inspect, sync-from, fetch-from, or delete a block's link to upstream content.
@@ -84,9 +91,16 @@ class UpstreamLink:
version_available: int | None # Latest version of the upstream that's available, or None if it couldn't be loaded.
version_declined: int | None # Latest version which the user has declined to sync with, if any.
error_message: str | None # If link is valid, None. Otherwise, a localized, human-friendly error message.
- is_modified: bool | None # If modified in course, True. Otherwise, False.
+ downstream_customized: list[str] | None # List of fields modified in downstream
has_top_level_parent: bool # True if this Upstream link has a top-level parent
+ @property
+ def is_upstream_deleted(self) -> bool:
+ return bool(
+ self.upstream_ref and
+ self.version_available is None
+ )
+
@property
def is_ready_to_sync_individually(self) -> bool:
return bool(
@@ -94,7 +108,7 @@ def is_ready_to_sync_individually(self) -> bool:
self.version_available and
self.version_available > (self.version_synced or 0) and
self.version_available > (self.version_declined or 0)
- )
+ ) or self.is_upstream_deleted
def _check_children_ready_to_sync(self, xblock_downstream: XBlock, return_fast: bool) -> list[dict[str, str]]:
"""
@@ -121,6 +135,8 @@ def _check_children_ready_to_sync(self, xblock_downstream: XBlock, return_fast:
child_info.append({
'name': child.display_name,
'upstream': getattr(child, 'upstream', None),
+ 'block_type': child.usage_key.block_type,
+ 'downstream_customized': child_upstream_link.downstream_customized,
'id': str(child.usage_key),
})
if return_fast:
@@ -180,6 +196,7 @@ def to_json(self, include_child_info=False) -> dict[str, t.Any]:
**asdict(self),
"ready_to_sync": self.ready_to_sync,
"upstream_link": self.upstream_link,
+ "is_ready_to_sync_individually": self.is_ready_to_sync_individually,
}
if (
include_child_info
@@ -219,7 +236,7 @@ def try_get_for_block(cls, downstream: XBlock, log_error: bool = True) -> t.Self
version_available=None,
version_declined=None,
error_message=str(exc),
- is_modified=len(getattr(downstream, "downstream_customized", [])) > 0,
+ downstream_customized=getattr(downstream, "downstream_customized", []),
has_top_level_parent=getattr(downstream, "top_level_downstream_parent_key", None) is not None,
)
@@ -302,7 +319,7 @@ def get_for_block(cls, downstream: XBlock) -> t.Self:
version_available=version_available,
version_declined=downstream.upstream_version_declined,
error_message=None,
- is_modified=len(getattr(downstream, "downstream_customized", [])) > 0,
+ downstream_customized=getattr(downstream, "downstream_customized", []),
has_top_level_parent=downstream.top_level_downstream_parent_key is not None,
)
@@ -337,7 +354,7 @@ def decline_sync(downstream: XBlock, user_id=None) -> None:
def _update_children_top_level_parent(
downstream: XBlock,
- new_top_level_parent_key: dict[str, str] | None
+ new_top_level_parent_key: str | None,
) -> list[XBlock]:
"""
Given a new top-level parent block, update the `top_level_downstream_parent_key` field on the downstream block
@@ -357,7 +374,7 @@ def _update_children_top_level_parent(
# If the `new_top_level_parent_key` is None, the current level assume the top-level
# parent key for its children.
child_top_level_parent_key = new_top_level_parent_key if new_top_level_parent_key is not None else (
- BlockKey.from_usage_key(child.usage_key)._asdict()
+ str(BlockKey.from_usage_key(child.usage_key))
)
affected_blocks.extend(_update_children_top_level_parent(child, child_top_level_parent_key))
@@ -433,7 +450,7 @@ class UpstreamSyncMixin(XBlockMixin):
"""
# Upstream synchronization metadata fields
- upstream = String(
+ upstream = _UpstreamKeyString(
help=(
"The usage key or container key of the source block/container (generally within a content library) "
"which serves as a source of upstream updates for this block, or None if there is no such upstream. "
@@ -466,7 +483,7 @@ class UpstreamSyncMixin(XBlockMixin):
default=None, scope=Scope.settings, hidden=True, enforce_type=True,
)
- top_level_downstream_parent_key = Dict(
+ top_level_downstream_parent_key = String(
help=(
"The block key ('block_type@block_id') of the downstream block that is the top-level parent of "
"this block. This is present if the creation of this block is a consequence of "
diff --git a/cms/static/images/large-games-icon.svg b/cms/static/images/large-games-icon.svg
new file mode 100644
index 000000000000..d23b38c7748d
--- /dev/null
+++ b/cms/static/images/large-games-icon.svg
@@ -0,0 +1,10 @@
+
diff --git a/cms/static/js/spec/views/modals/edit_xblock_spec.js b/cms/static/js/spec/views/modals/edit_xblock_spec.js
index 06b4413784cb..0855a850b97d 100644
--- a/cms/static/js/spec/views/modals/edit_xblock_spec.js
+++ b/cms/static/js/spec/views/modals/edit_xblock_spec.js
@@ -185,6 +185,68 @@ describe('EditXBlockModal', function() {
});
});
+ describe('isSafeReturnTo', function() {
+ var isSafeReturnTo = EditXBlockModal.isSafeReturnTo;
+
+ // -- valid inputs ---------------------------------------------------
+
+ it('accepts a root-relative path', function() {
+ expect(isSafeReturnTo('/course/123')).toBe(true);
+ });
+
+ it('accepts the root path', function() {
+ expect(isSafeReturnTo('/')).toBe(true);
+ });
+
+ it('accepts a root-relative path with query string', function() {
+ expect(isSafeReturnTo('/course/123?tab=outline')).toBe(true);
+ });
+
+ it('accepts an absolute URL with the same origin', function() {
+ expect(isSafeReturnTo(window.location.origin + '/course/123')).toBe(true);
+ });
+
+ // -- empty / missing values -----------------------------------------
+
+ it('rejects an empty string', function() {
+ expect(isSafeReturnTo('')).toBe(false);
+ });
+
+ it('rejects null', function() {
+ expect(isSafeReturnTo(null)).toBe(false);
+ });
+
+ it('rejects undefined', function() {
+ expect(isSafeReturnTo(undefined)).toBe(false);
+ });
+
+ it('rejects a non-string value', function() {
+ expect(isSafeReturnTo(42)).toBe(false);
+ });
+
+ // -- protocol-relative / different origin ---------------------------
+
+ it('rejects protocol-relative URLs', function() {
+ expect(isSafeReturnTo('//evil.com/path')).toBe(false);
+ });
+
+ it('rejects an absolute URL with a different origin', function() {
+ expect(isSafeReturnTo('https://evil.com/steal')).toBe(false);
+ });
+
+ // -- relative paths without leading / -------------------------------
+
+ it('rejects a bare relative path', function() {
+ expect(isSafeReturnTo('course/123')).toBe(false);
+ });
+
+ // -- javascript: scheme ---------------------------------------------
+
+ it('rejects javascript: scheme', function() {
+ expect(isSafeReturnTo('javascript:alert(1)')).toBe(false); // eslint-disable-line no-script-url
+ });
+ });
+
describe('XModule Editor (settings only)', function() {
var mockXModuleEditorHtml;
diff --git a/cms/static/js/views/modals/edit_xblock.js b/cms/static/js/views/modals/edit_xblock.js
index 586d27d8b284..aef409c56150 100644
--- a/cms/static/js/views/modals/edit_xblock.js
+++ b/cms/static/js/views/modals/edit_xblock.js
@@ -8,6 +8,27 @@ define(['jquery', 'underscore', 'backbone', 'gettext', 'js/views/modals/base_mod
function($, _, Backbone, gettext, BaseModal, ViewUtils, XBlockViewUtils, XBlockEditorView) {
'use strict';
+ /**
+ * Returns true when ``url`` is safe to use as a same-origin redirect
+ * destination. Accepted values are:
+ * - Root-relative paths beginning with '/' but NOT '//' (protocol-
+ * relative URLs such as //evil.com would bypass the check).
+ * - Absolute URLs whose origin matches the current page's origin.
+ */
+ function isSafeReturnTo(url) {
+ if (typeof url !== 'string' || url.length === 0) {
+ return false;
+ }
+ if (url.charAt(0) === '/' && url.charAt(1) !== '/') {
+ return true;
+ }
+ try {
+ return new URL(url).origin === window.location.origin;
+ } catch (e) {
+ return false;
+ }
+ }
+
var EditXBlockModal = BaseModal.extend({
events: _.extend({}, BaseModal.prototype.events, {
'click .action-save': 'save',
@@ -234,6 +255,13 @@ function($, _, Backbone, gettext, BaseModal, ViewUtils, XBlockViewUtils, XBlockE
console.error(e);
}
+ var returnTo = this.editOptions && this.editOptions.returnTo;
+ if (returnTo && isSafeReturnTo(returnTo)) {
+ this.hide();
+ window.location.href = returnTo;
+ return;
+ }
+
var refresh = this.editOptions.refresh;
this.hide();
if (refresh) {
@@ -241,6 +269,18 @@ function($, _, Backbone, gettext, BaseModal, ViewUtils, XBlockViewUtils, XBlockE
}
},
+ cancel: function(event) {
+ if (event) {
+ event.preventDefault();
+ event.stopPropagation();
+ }
+ this.hide();
+ var returnTo = this.editOptions && this.editOptions.returnTo;
+ if (returnTo && isSafeReturnTo(returnTo)) {
+ window.location.href = returnTo;
+ }
+ },
+
hide: function() {
// Notify child views to stop listening events
Backbone.trigger('xblock:editorModalHidden');
@@ -296,5 +336,8 @@ function($, _, Backbone, gettext, BaseModal, ViewUtils, XBlockViewUtils, XBlockE
});
+ // Expose for unit testing.
+ EditXBlockModal.isSafeReturnTo = isSafeReturnTo;
+
return EditXBlockModal;
});
diff --git a/cms/static/js/views/pages/container.js b/cms/static/js/views/pages/container.js
index d50f6b4bbe4a..93095745f162 100644
--- a/cms/static/js/views/pages/container.js
+++ b/cms/static/js/views/pages/container.js
@@ -305,7 +305,9 @@ function($, _, Backbone, gettext, BasePage,
renderAddXBlockComponents: function() {
var self = this;
- if (self.options.canEdit && (!self.options.isIframeEmbed || self.isSplitTestContentPage)) {
+ // If the container is the Unit element(aka Vertical), then we don't render the
+ // add buttons because those should get rendered by the authoring MFE
+ if (self.options.canEdit && (!self.options.isIframeEmbed || !self.model.isVertical())) {
this.$('.add-xblock-component').each(function(index, element) {
var component = new AddXBlockComponent({
el: element,
@@ -504,14 +506,14 @@ function($, _, Backbone, gettext, BasePage,
if (!options || options.view !== 'visibility_view') {
const primaryHeader = $(event.target).closest('.xblock-header-primary, .nav-actions');
- var useNewTextEditor = primaryHeader.attr('use-new-editor-text'),
- useNewVideoEditor = primaryHeader.attr('use-new-editor-video'),
+ var useNewVideoEditor = primaryHeader.attr('use-new-editor-video'),
useNewProblemEditor = primaryHeader.attr('use-new-editor-problem'),
blockType = primaryHeader.attr('data-block-type');
- if((useNewTextEditor === 'True' && blockType === 'html')
+ if((blockType === 'html')
|| (useNewVideoEditor === 'True' && blockType === 'video')
|| (useNewProblemEditor === 'True' && blockType === 'problem')
+ || (blockType === 'games') || (blockType === 'invideoquiz')
) {
var destinationUrl = primaryHeader.attr('authoring_MFE_base_url')
+ '/' + blockType
@@ -633,13 +635,13 @@ function($, _, Backbone, gettext, BasePage,
doneAddingBlock = (addResult) => {
const $placeholderEl = $(this.createPlaceholderElement());
const placeholderElement = $placeholderEl.insertBefore($insertSpot);
- placeholderElement.data('locator', addResult.locator);
- return this.refreshXBlock(placeholderElement, true);
+ return this.onNewXBlock(placeholderElement, 0, false, addResult);
};
doneAddingAllBlocks = () => {};
}
// Note: adding all the XBlocks in parallel will cause a race condition 😢 so we have to add
// them one at a time:
+
let lastAdded = $.when();
for (const { usageKey, blockType } of selectedBlocks) {
const addData = {
@@ -1170,8 +1172,7 @@ function($, _, Backbone, gettext, BasePage,
},
onNewXBlock: function(xblockElement, scrollOffset, is_duplicate, data) {
- var useNewTextEditor = this.$('.xblock-header-primary').attr('use-new-editor-text'),
- useNewVideoEditor = this.$('.xblock-header-primary').attr('use-new-editor-video'),
+ var useNewVideoEditor = this.$('.xblock-header-primary').attr('use-new-editor-video'),
useVideoGalleryFlow = this.$('.xblock-header-primary').attr("use-video-gallery-flow"),
useNewProblemEditor = this.$('.xblock-header-primary').attr('use-new-editor-problem');
@@ -1181,9 +1182,11 @@ function($, _, Backbone, gettext, BasePage,
var blockType = data.locator.match(matchBlockTypeFromLocator);
}
// open mfe editors for new blocks only and not for content imported from libraries
- if(!data.hasOwnProperty('upstreamRef') && ((useNewTextEditor === 'True' && blockType.includes('html'))
+ if(!data.hasOwnProperty('upstreamRef') && (blockType.includes('html')
|| (useNewVideoEditor === 'True' && blockType.includes('video'))
- || (useNewProblemEditor === 'True' && blockType.includes('problem')))
+ || (useNewProblemEditor === 'True' && blockType.includes('problem'))
+ || blockType.includes('games')
+ || blockType.includes('invideoquiz'))
){
if (this.options.isIframeEmbed && (this.isSplitTestContentPage || this.isVerticalContentPage)) {
return this.postMessageToParent({
@@ -1222,12 +1225,13 @@ function($, _, Backbone, gettext, BasePage,
refreshXBlock: function(element, block_added, is_duplicate) {
var xblockElement = this.findXBlockElement(element),
parentElement = xblockElement.parent(),
- rootLocator = this.xblockView.model.id;
+ rootLocator = this.xblockView.model.id,
+ parentBlockType = parentElement.data('block-type');
if (xblockElement.length === 0 || xblockElement.data('locator') === rootLocator) {
if (block_added) {
this.render({refresh: true, block_added: block_added});
}
- } else if (parentElement.hasClass('reorderable-container')) {
+ } else if (parentElement.hasClass('reorderable-container') || ["itembank", "library_content"].includes(parentBlockType) ) {
this.refreshChildXBlock(xblockElement, block_added, is_duplicate);
} else {
this.refreshXBlock(this.findXBlockElement(parentElement));
diff --git a/cms/static/sass/_build-v1.scss b/cms/static/sass/_build-v1.scss
index 178f6b167473..7aedd0c6b8a3 100644
--- a/cms/static/sass/_build-v1.scss
+++ b/cms/static/sass/_build-v1.scss
@@ -77,7 +77,6 @@
@import 'views/group-configuration';
@import 'views/video-upload';
@import 'views/certificates';
-@import 'views/maintenance';
// +Base - Contexts
// ====================
diff --git a/cms/static/sass/assets/_graphics.scss b/cms/static/sass/assets/_graphics.scss
index afb830d5dd71..58141c445a75 100644
--- a/cms/static/sass/assets/_graphics.scss
+++ b/cms/static/sass/assets/_graphics.scss
@@ -80,3 +80,9 @@
height: ($baseline*3);
background: url('#{$static-path}/images/large-itembank-icon.png') center no-repeat;
}
+
+.large-games-icon {
+ display: inline-block;
+ width: ($baseline*3);
+ height: ($baseline*3);
+ background: url('#{$static-path}/images/large-games-icon.svg') center no-repeat; }
diff --git a/cms/static/sass/elements/_modules.scss b/cms/static/sass/elements/_modules.scss
index 1e9f52691fc8..f1c871ba1d95 100644
--- a/cms/static/sass/elements/_modules.scss
+++ b/cms/static/sass/elements/_modules.scss
@@ -163,8 +163,8 @@
display: inline-block;
color: $uxpl-green-base;
- background-color: theme-color("inverse");
- border-color: theme-color("inverse");
+ background-color: $white;
+ border-color: $white;
border-radius: 3px;
font-size: 90%;
diff --git a/cms/static/sass/views/_maintenance.scss b/cms/static/sass/views/_maintenance.scss
deleted file mode 100644
index 58d1b7494751..000000000000
--- a/cms/static/sass/views/_maintenance.scss
+++ /dev/null
@@ -1,104 +0,0 @@
-.maintenance-header {
- text-align: center;
- margin-top: 50px;
-
- h2 {
- margin-bottom: 10px;
- }
-}
-
-.maintenance-content {
- padding: 3rem 0;
-
- .maintenance-list {
- max-width: 1280px;
- margin: 0 auto;
-
- .view-list-container {
- padding: 10px 15px;
- background-color: #fff;
- border-bottom: 1px solid #ddd;
-
- &:hover {
- background-color: #fafafa;
- }
-
- .view-name {
- display: inline-block;
- width: 20%;
- float: left;
- }
-
- .view-desc {
- display: inline-block;
- width: 80%;
- font-size: 15px;
- }
- }
- }
-
- .maintenance-form {
- width: 60%;
- margin: auto;
-
- .result-list {
- height: calc(100vh - 200px);
- overflow: auto;
- }
-
- .result {
- box-shadow: inset 0 1px 2px rgba(0, 0, 0, 0.2);
- margin-top: 15px;
- padding: 15px 30px;
- background: #f9f9f9;
- }
-
- li {
- font-size: 13px;
- line-height: 9px;
- }
-
- .actions {
- text-align: right;
- }
-
- .field-radio div {
- display: inline-block;
- margin-right: 10px;
- }
-
- div.error {
- color: #f00;
- margin-top: 10px;
- font-size: 13px;
- }
-
- div.head-output {
- font-size: 13px;
- margin-bottom: 10px;
- }
-
- div.main-output {
- color: #0a0;
- font-size: 15px;
- }
- }
-
- .announcement-container {
- width: 100%;
- text-align: center;
-
- .announcement-item {
- display: inline-block;
- max-width: 300px;
- min-width: 300px;
- margin: 15px;
-
- .announcement-content {
- background-color: $body-bg;
- text-align: center;
- padding: 22px 33px;
- }
- }
- }
-}
diff --git a/cms/static/sass/xmodule/_headings.scss b/cms/static/sass/xmodule/_headings.scss
index 07ae25606db4..a24681fc81e7 100644
--- a/cms/static/sass/xmodule/_headings.scss
+++ b/cms/static/sass/xmodule/_headings.scss
@@ -41,7 +41,7 @@ $headings-base-color: $gray-d2;
%hd-3 {
margin-bottom: ($baseline / 2);
font-size: 1.35em;
- font-weight: $headings-font-weight-normal;
+ font-weight: $headings-font-weight-bold;
line-height: 1.4em;
}
@@ -105,15 +105,13 @@ $headings-base-color: $gray-d2;
// ----------------------------
// canned heading classes
@for $i from 1 through $headings-count {
+ h#{$i},
.hd-#{$i} {
@extend %hd-#{$i};
}
}
h3 {
- @extend %hd-2;
-
- font-weight: $headings-font-weight-normal;
// override external modules and xblocks that use inline CSS
text-transform: initial;
}
diff --git a/cms/templates/accessibility.html b/cms/templates/accessibility.html
deleted file mode 100644
index bb7a13f11441..000000000000
--- a/cms/templates/accessibility.html
+++ /dev/null
@@ -1,37 +0,0 @@
-<%page expression_filter="h"/>
-<%inherit file="base.html" />
-<%def name="online_help_token()"><% return "accessibility" %>
-<%!
- from django.urls import reverse
- from django.utils.translation import gettext as _
- from openedx.core.djangolib.markup import HTML, Text
- from openedx.core.djangolib.js_utils import js_escaped_string, dump_js_escaped_json
-%>
-<%block name="title">${_("Studio Accessibility Policy")}
-<%block name="bodyclass">is-signedin not-signedin view-accessibility
-
-<%namespace name='static' file='static_content.html'/>
-
-<%block name="header_extras">
- % if not settings.STUDIO_FRONTEND_CONTAINER_URL:
-
-
- % endif
-
-
-<%block name="content">
-
-
See image: referenced.png
See image: referenced.png
See image: duplicate.png
-
-
-
diff --git a/cms/templates/asset_index.html b/cms/templates/asset_index.html
deleted file mode 100644
index fc7d92173b0c..000000000000
--- a/cms/templates/asset_index.html
+++ /dev/null
@@ -1,59 +0,0 @@
-<%page expression_filter="h"/>
-<%inherit file="base.html" />
-<%def name="online_help_token()"><% return "files" %>
-<%!
- from django.urls import reverse
- from django.utils.translation import gettext as _
- from openedx.core.djangolib.markup import HTML, Text
- from openedx.core.djangolib.js_utils import js_escaped_string, dump_js_escaped_json
-%>
-<%block name="title">${_("Files")}
-<%block name="bodyclass">is-signedin course uploads view-uploads
-
-<%namespace name='static' file='static_content.html'/>
-
-<%block name="header_extras">
- % if not settings.STUDIO_FRONTEND_CONTAINER_URL:
-
-
- % endif
-
-
-<%block name="content">
-
-
-
-
-
- <%static:studiofrontend entry="accessibilityPolicy">
- {
- "lang": "${language_code | n, js_escaped_string}"
- }
-
-
-
-
-
-
-
-- ${_("Content")} - - ${_("Files")} -
-
-
-
-
diff --git a/cms/templates/checklists.html b/cms/templates/checklists.html
deleted file mode 100644
index e1859f38f70b..000000000000
--- a/cms/templates/checklists.html
+++ /dev/null
@@ -1,75 +0,0 @@
-<%page expression_filter="h"/>
-<%inherit file="base.html" />
-<%def name="online_help_token()"><% return "files" %>
-<%!
- from cms.djangoapps.contentstore import utils
- from cms.djangoapps.contentstore.config.waffle_utils import should_show_checklists_quality
- from django.urls import reverse
- from django.utils.translation import gettext as _
- from openedx.core.djangolib.markup import HTML, Text
- from openedx.core.djangolib.js_utils import js_escaped_string, dump_js_escaped_json
- from common.djangoapps.util.course import has_certificates_enabled
-%>
-<%block name="title">${_("Checklists")}
-<%block name="bodyclass">is-signedin course view-checklists
-
-<%namespace name='static' file='static_content.html'/>
-
-<%block name="header_extras">
- % if not settings.STUDIO_FRONTEND_CONTAINER_URL:
-
-
- % endif
-
-
-<%block name="content">
-
-
- <%static:optional_include_mako file="asset_index_content_header.html" />
- <%static:studiofrontend entry="assets">
- {
- "lang": "${language_code | n, js_escaped_string}",
- "course": {
- "id": "${context_course.id | n, js_escaped_string}",
- "name": "${context_course.display_name_with_default | n, js_escaped_string}",
- "url_name": "${context_course.location.block_id | n, js_escaped_string}",
- "org": "${context_course.location.org | n, js_escaped_string}",
- "num": "${context_course.location.course | n, js_escaped_string}",
- "display_course_number": "${context_course.display_coursenumber | n, js_escaped_string}",
- "revision": "${context_course.location.branch | n, js_escaped_string}"
- },
- "help_tokens": {
- "files": "${get_online_help_info(online_help_token())['doc_url'] | n, js_escaped_string}"
- },
- "upload_settings": {
- "max_file_size_in_mbs": ${max_file_size_in_mbs|n, dump_js_escaped_json}
- }
- }
-
-
-
-
-
-
-
-- ${_("Tools")} - - ${_("Checklists")} -
-
-
-
-
diff --git a/cms/templates/container.html b/cms/templates/container.html
index 1c56740ca9e3..61e52f617ab0 100644
--- a/cms/templates/container.html
+++ b/cms/templates/container.html
@@ -13,7 +13,7 @@
from django.utils.translation import gettext as _
from cms.djangoapps.contentstore.helpers import xblock_studio_url, xblock_type_display_name
-from cms.djangoapps.contentstore.toggles import use_new_text_editor, use_new_problem_editor, use_new_video_editor, use_video_gallery_flow
+from cms.djangoapps.contentstore.toggles import use_new_problem_editor, use_new_video_editor, use_video_gallery_flow
from cms.djangoapps.contentstore.utils import get_editor_page_base_url
from openedx.core.djangolib.js_utils import (
dump_js_escaped_json, js_escaped_string
@@ -36,10 +36,6 @@
<%static:include path="common/templates/image-modal.underscore" />
-% if not settings.STUDIO_FRONTEND_CONTAINER_URL:
-
-
-% endif
<%block name="page_bundle">
@@ -115,7 +111,6 @@
<%block name="content">
<%
-use_new_editor_text = use_new_text_editor(xblock_locator.course_key)
use_new_editor_video = use_new_video_editor(xblock_locator.course_key)
use_new_editor_problem = use_new_problem_editor(xblock_locator.course_key)
use_new_video_gallery_flow = use_video_gallery_flow()
@@ -172,7 +167,6 @@
- <%static:studiofrontend entry="courseHealthCheck">
-
- <%
- course_key = str(context_course.id)
- certificates_url = ''
- if has_certificates_enabled(context_course):
- certificates_url = utils.reverse_course_url('certificates_list_handler', course_key)
- %>
- {
- "lang": "${language_code | n, js_escaped_string}",
- "course": {
- "id": "${context_course.id | n, js_escaped_string}",
- "name": "${context_course.display_name_with_default | n, js_escaped_string}",
- "is_course_self_paced": ${context_course.self_paced | n, dump_js_escaped_json},
- "url_name": "${context_course.location.block_id | n, js_escaped_string}",
- "org": "${context_course.location.org | n, js_escaped_string}",
- "num": "${context_course.location.course | n, js_escaped_string}",
- "display_course_number": "${context_course.display_coursenumber | n, js_escaped_string}",
- "revision": "${context_course.location.revision | n, js_escaped_string}"
- },
- "help_tokens": {
- "files": "${get_online_help_info(online_help_token())['doc_url'] | n, js_escaped_string}"
- },
- "enable_quality": ${should_show_checklists_quality(context_course.id) | n, dump_js_escaped_json},
- "links": {
- "certificates": ${certificates_url | n, dump_js_escaped_json},
- "course_outline": ${utils.reverse_course_url('course_handler', course_key) | n, dump_js_escaped_json},
- "course_updates": ${utils.reverse_course_url('course_info_handler', course_key) | n, dump_js_escaped_json},
- "grading_policy": ${utils.reverse_course_url('grading_handler', course_key) | n, dump_js_escaped_json},
- "settings": ${utils.reverse_course_url('settings_handler', course_key) | n, dump_js_escaped_json},
- "proctored_exam_settings": ${mfe_proctored_exam_settings_url | n, dump_js_escaped_json}
- }
- }
-
-
-
-
- %endif
-
- %if context_course.discussions_settings.get('provider_type') == "openedx":
-
-
-
-
-
-
-
-
- ${_("This course was created as a re-run. Some manual configuration is needed.")}
- -${_("No course content is currently visible, and no learners are enrolled. Be sure to review and reset all dates, including the Course Start Date; set up the course team; review course updates and other assets for dated material; and seed the discussions and wiki.")}
-
-
- %endif
-
-
- %if deprecated_blocks_info.get('blocks') or deprecated_blocks_info.get('deprecated_enabled_block_types'):
-
-
-
-
-
-
-
-
-
-
- ${_("This course run is using an upgraded version of edx discussion forum. In order to display the discussions sidebar, discussions xBlocks will no longer be visible to learners.")}
-
-
- %if settings.DISCUSSIONS_INCONTEXT_LEARNMORE_URL:
-
- ${_(" Learn more")}
-
-
- %endif
- %if settings.DISCUSSIONS_INCONTEXT_FEEDBACK_URL:
-
- ${_("Share feedback")}
-
-
- %endif
-
-
-
- %endif
-
- %if proctoring_errors:
-
- ${_("Warning")}
-
-
-
-
- ${_("This course uses features that are no longer supported.")}
- - %if deprecated_blocks_info.get('blocks'): -
-
- %endif
-
- % if deprecated_blocks_info.get('deprecated_enabled_block_types'):
- ${_("You must delete or replace the following components.")}
- -
-
- % endif
- - ${Text(_("To avoid errors, {platform_name} strongly recommends that you remove unsupported features from the course advanced settings. To do this, go to the {link_start}Advanced Settings page{link_end}, locate the \"Advanced Module List\" setting, and then delete the following modules from the list.")).format( - platform_name=static.get_platform_name(), - link_start=HTML('').format(advance_settings_url=deprecated_blocks_info['advance_settings_url']), - link_end=HTML("") - )} -
- -
-
- %endif
-
-
-
-<%block name="content">
-
-
-
-
-
-
- ${_("This course has proctored exam settings that are incomplete or invalid.")}
-- % if mfe_proctored_exam_settings_url: - <% url_encoded_course_id = quote(str(context_course.id).encode('utf-8'), safe='') %> - ${Text(_("To update these settings go to the {link_start}Proctored Exam Settings page{link_end}.")).format( - link_start=HTML('').format( - mfe_proctored_exam_settings_url=mfe_proctored_exam_settings_url - ), - link_end=HTML("") - )} - % else: - ${Text(_("To update these settings go to the {link_start}Advanced Settings page{link_end}.")).format( - link_start=HTML('').format(advance_settings_url=advance_settings_url), - link_end=HTML("") - )} - % endif -
-
-
-
-
-
-
-
-
-- ${_("Content")} - > ${_("Course Outline")} -
- - -
-
-
-
diff --git a/cms/templates/maintenance/_announcement_delete.html b/cms/templates/maintenance/_announcement_delete.html
deleted file mode 100644
index 0397ef5a0bf8..000000000000
--- a/cms/templates/maintenance/_announcement_delete.html
+++ /dev/null
@@ -1,40 +0,0 @@
-<%page expression_filter="h"/>
-<%inherit file="base.html" />
-<%namespace name='static' file='../static_content.html'/>
-<%!
-from django.utils.translation import gettext as _
-from openedx.core.djangolib.markup import HTML, Text
-%>
-<%block name="title">${_('Delete Announcement')}
-<%block name="viewtitle">
-
- ## set width dynamically depending upon whether course has a start date to ensure spacing looks good
- % if course_release_date == 'Set Date':
-
-
-
-
-
diff --git a/cms/templates/index.html b/cms/templates/index.html
deleted file mode 100644
index e55859730729..000000000000
--- a/cms/templates/index.html
+++ /dev/null
@@ -1,629 +0,0 @@
-<%page expression_filter="h"/>
-<%!
-from django.utils.translation import gettext as _
-from django.urls import reverse
-
-from openedx.core.djangolib.markup import HTML, Text
-from openedx.core.djangolib.js_utils import (
- dump_js_escaped_json
- )
-%>
-
-<%inherit file="base.html" />
-<%namespace name='static' file='static_content.html'/>
-<%def name="online_help_token()"><% return "home" %>
-<%block name="title">${_("{studio_name} Home").format(studio_name=settings.STUDIO_SHORT_NAME)}
-<%block name="bodyclass">is-signedin index view-dashboard
-
-<%block name="requirejs">
- require(["js/factories/index"], function (IndexFactory) {
- IndexFactory();
- });
-
-
-<%block name="content">
-
- % else:
-
-
- % endif
- <%static:studiofrontend entry="courseOutlineHealthCheck">
- <%
- course_key = context_course.id
- %>
- {
- "lang": "${language_code | n, js_escaped_string}",
- "course": {
- "id": "${context_course.id | n, js_escaped_string}",
- "name": "${context_course.display_name_with_default | n, js_escaped_string}",
- "course_release_date": "${course_release_date | n, js_escaped_string}",
- "is_course_self_paced": ${context_course.self_paced | n, dump_js_escaped_json},
- "url_name": "${context_course.location.block_id | n, js_escaped_string}",
- "org": "${context_course.location.org | n, js_escaped_string}",
- "num": "${context_course.location.course | n, js_escaped_string}",
- "display_course_number": "${context_course.display_coursenumber | n, js_escaped_string}",
- "revision": "${context_course.location.branch | n, js_escaped_string}"
- },
- "help_tokens": {
- "files": "${get_online_help_info(online_help_token())['doc_url'] | n, js_escaped_string}"
- },
- "enable_quality": ${should_show_checklists_quality(context_course.id) | n, dump_js_escaped_json},
- "links": {
- "settings": ${reverse('settings_handler', kwargs={'course_key_string': str(course_key)})| n, dump_js_escaped_json}
- }
- }
-
-
-
-
-
-
- <%
- course_locator = context_course.location
- assets_url = reverse('assets_handler', kwargs={'course_key_string': str(course_locator.course_key)})
- %>
-
-
-
- ${_("Course Outline")}
-
-
-
-
-
-${_("Loading")}
-
-
-
-
-
-${_("{studio_name} Home").format(studio_name=settings.STUDIO_SHORT_NAME)}
- - % if user.is_active: - - % endif -
- % if user.is_active:
-
-
-
- % if course_creator_status=='granted':
-
-
-
-
-
- % else:
-
-
-
-
-
-
-
- %endif
-
-
diff --git a/cms/templates/js/show-correctness-editor.underscore b/cms/templates/js/show-correctness-editor.underscore
index 3db6c3c27a5c..1b0dd896747a 100644
--- a/cms/templates/js/show-correctness-editor.underscore
+++ b/cms/templates/js/show-correctness-editor.underscore
@@ -35,6 +35,13 @@
<% } %>
<%- gettext('If the subsection does not have a due date, learners always see their scores when they submit answers to assessments.') %>
+
+
-
-
-
- % endif
-
- %if libraries_enabled and show_new_library_button:
-
-
-
- % endif
-
- %if optimization_enabled:
-
-
- %endif
-
-
- %if allow_course_reruns and rerun_creator_status and len(in_process_course_actions) > 0:
- ${_("Organization and Library Settings")}
- -
-
- %endif
-
- % if libraries_enabled or archived_courses:
- ${_("Courses Being Processed")}
- --
- %for course_info in sorted(in_process_course_actions, key=lambda s: s['display_name'].lower() if s['display_name'] is not None else ''):
-
- %if course_info['is_in_progress']:
-
-
- -- --- -
${course_info['display_name']}
- - --
-
- ${_("This course run is currently being created.")} -
- - - ## Translators: This is a status message, used to inform the user of - ## what the system is doing. This status means that the user has - ## requested to re-run an existing course, and the system is currently - ## in the process of duplicating and configuring the existing course - ## so that it can be re-run. - ${_("Configuring as re-run")} - -
--${Text(_('The new course will be added to your course list in 5-10 minutes. Return to this page or {link_start}refresh it{link_end} to update the course list. The new course will need some manual configuration.')).format( - link_start=HTML(''), - link_end=HTML(''), - )}
-
- %endif
-
-
-
-
- %if course_info['is_failed']:
- -
- -- --- -
${course_info['display_name']}
- - --
- ## Translators: This is a status message for the course re-runs feature.
- ## When a course admin indicates that a course should be re-run, the system
- ## needs to process the request and prepare the new course. The status of
- ## the process will follow this text.
-
- ${_("This re-run processing status:")} -
- - - ${_("Configuration Error")} - -
--${_("A system error occurred while your course was being processed. Please go to the original course to try the re-run again, or contact your PM for assistance.")}
- --
-
- - - - - - -
- %endif
- %endfor
-
-
-
- - % if split_studio_home and active_tab == 'libraries': - ${_("Courses")} - % else: - ${_("Courses")} - % endif - - % if archived_courses: -
- - % if split_studio_home and active_tab == 'libraries': - ${_("Archived Courses")} - % else: - ${_("Archived Courses")} - % endif - - % endif - - % if libraries_enabled: -
- - % if split_studio_home: - ${_("Libraries")} - % else: - ${_("Libraries")} - % endif - - % endif - % if taxonomies_enabled: -
- ${_("Taxonomies")} - % endif -
- ${static.renderReact(
- component="CourseOrLibraryListing",
- id="react-course-listing",
- props={
- 'items': sorted(courses, key=lambda s: s['display_name'].lower() if s['display_name'] is not None else ''),
- 'linkClass': 'course-link',
- 'idBase': 'course',
- 'allowReruns': allow_course_reruns and rerun_creator_status and course_creator_status=='granted'
- }
- )}
-
-
- %else:
-
-
- % endif
-
-
- %if course_creator_status == "unrequested":
-
-
-
- %if course_creator_status == "granted":
-
-
- ${_("Are you staff on an existing {studio_name} course?").format(studio_name=settings.STUDIO_SHORT_NAME)}
-
-
- ${_('The course creator must give you access to the course. Contact the course creator or administrator for the course you are helping to author.')}
-
-
- % endif
-
-
-
-
- ${_('Create Your First Course')}
-
-
- ${_('Your new course is just a click away!')}
--
-
- - ${_('Create Your First Course')} - -
-
-
- %elif course_creator_status == "denied":
- - ${_('Becoming a Course Creator in {studio_name}').format(studio_name=settings.STUDIO_SHORT_NAME)} -
- -
-
-
-
-
- ${_('{studio_name} is a hosted solution for our xConsortium partners and selected guests. Courses for which you are a team member appear above for you to edit, while course creator privileges are granted by {platform_name}. Our team will evaluate your request and provide you feedback within 24 hours during the work week.').format( - studio_name=settings.STUDIO_NAME, platform_name=settings.PLATFORM_NAME)}
-
-
- ${_('Your Course Creator Request Status:')}
- - -
-
-
- %elif course_creator_status == "pending":
- - ${_('Your Course Creator Request Status')} -
- -
-
-
-
-
- ${_('{studio_name} is a hosted solution for our xConsortium partners and selected guests. Courses for which you are a team member appear above for you to edit, while course creator privileges are granted by {platform_name}. Our team is has completed evaluating your request.').format( - studio_name=settings.STUDIO_NAME, platform_name=settings.PLATFORM_NAME, - )}
-
-
- ${_('Your Course Creator Request Status:')}
- --
-
- ${_('Your Course Creator request is:')} -
- - - ${_('Denied')} - ${_('Your request did not meet the criteria/guidelines specified by {platform_name} Staff.').format(platform_name=settings.PLATFORM_NAME)} - -
-
- % endif
-
- %if archived_courses:
- - ${_('Your Course Creator Request Status')} -
- -
-
-
-
-
- ${_('{studio_name} is a hosted solution for our xConsortium partners and selected guests. Courses for which you are a team member appear above for you to edit, while course creator privileges are granted by {platform_name}. Our team is currently evaluating your request.').format( - studio_name=settings.STUDIO_NAME, platform_name=settings.PLATFORM_NAME, - )}
-
-
- ${_('Your Course Creator Request Status:')}
- --
-
- ${_('Your Course Creator request is:')} -
- - - ${_('Pending')} - - ${_('Your request is currently being reviewed by {platform_name} staff and should be updated shortly.').format(platform_name=settings.PLATFORM_NAME)} - - -
- % if type(archived_courses) is list:
- ${static.renderReact(
- component="CourseOrLibraryListing",
- id="react-archived-course-listing",
- props={
- 'items': sorted(archived_courses, key=lambda s: s['display_name'].lower() if s['display_name'] is not None else ''),
- 'linkClass': 'course-link',
- 'idBase': 'archived',
- 'allowReruns': allow_course_reruns and rerun_creator_status and course_creator_status=='granted'
- }
- )}
- % endif
-
- %endif
-
- %if len(libraries) > 0 or optimization_enabled:
-
- ${static.renderReact(
- component="CourseOrLibraryListing",
- id="react-library-listing",
- props={
- 'items': sorted(libraries, key=lambda s: s['display_name'].lower() if s['display_name'] is not None else ''),
- 'linkClass': 'library-link',
- 'idBase': 'library',
- 'allowReruns': allow_course_reruns and rerun_creator_status and course_creator_status=='granted'
- }
- )}
-
-
- %else:
-
-
- %endif
-
-
-
- % if show_new_library_button:
-
-
- ${_("Were you expecting to see a particular library here?")}
-
-
- ${_('The library creator must give you access to the library. Contact the library creator or administrator for the library you are helping to author.')}
-
-
- % endif
-
-
-
-
- ${_('Create Your First Library')}
-
-
- ${_('Libraries hold a pool of components that can be re-used across multiple courses. Create your first library with the click of a button!')}
-
-
-
- ${_("Thanks for signing up, {name}!").format(name=user.username)}
-
-
-
-
- ${_("We need to verify your email address")}
-
-
- ${_('Almost there! In order to complete your sign up we need you to verify your email address ({email}). An activation message and next steps should be waiting for you there.').format(email=user.email)}
-+ <%- gettext('Learners do not see question-level correctness or scores before or after the due date. However, once the due date passes, they can see their overall score for the subsection on the Progress page.') %> +
- ${_('Delete Announcement')} -
- - -<%block name="viewcontent"> -
-
-
-
-
-
- - ${_('Edit Announcement')} -
- - -<%block name="viewcontent"> -
-
-
-
-
diff --git a/cms/templates/maintenance/base.html b/cms/templates/maintenance/base.html
deleted file mode 100644
index 6979797a629c..000000000000
--- a/cms/templates/maintenance/base.html
+++ /dev/null
@@ -1,21 +0,0 @@
-<%page expression_filter="h"/>
-<%inherit file="../base.html" />
-<%def name='online_help_token()'><% return 'maintenance' %>
-<%namespace name='static' file='../static_content.html'/>
-<%!
-from django.urls import reverse
-from django.utils.translation import gettext as _
-%>
-<%block name="content">
-
-
-
- % for announcement in announcement_list:
-
- % endfor
-
-
-
-
- <%include file="_${view['slug']}.html"/>
-
-
-
-<%block name="requirejs">
- require(["js/maintenance/${view['slug'] | n, js_escaped_string}"], function(MaintenanceFactory) {
- MaintenanceFactory("${reverse(view['url']) | n, js_escaped_string}");
- });
-
diff --git a/cms/templates/maintenance/index.html b/cms/templates/maintenance/index.html
deleted file mode 100644
index 293cb90b4a9c..000000000000
--- a/cms/templates/maintenance/index.html
+++ /dev/null
@@ -1,20 +0,0 @@
-<%page expression_filter="h"/>
-<%inherit file="base.html" />
-<%namespace name='static' file='../static_content.html'/>
-<%!
-from django.utils.translation import gettext as _
-from django.urls import reverse
-%>
-<%block name="title">${_('Maintenance Dashboard')}
-<%block name="viewcontent">
-
-
-<%block name="viewcontent">
-
diff --git a/cms/templates/maintenance/container.html b/cms/templates/maintenance/container.html
deleted file mode 100644
index 319a57bfe995..000000000000
--- a/cms/templates/maintenance/container.html
+++ /dev/null
@@ -1,25 +0,0 @@
-<%page expression_filter="h"/>
-<%inherit file="base.html" />
-<%namespace name='static' file='../static_content.html'/>
-<%!
-from django.urls import reverse
-from openedx.core.djangolib.js_utils import js_escaped_string
-%>
-<%block name="title">${view['name']}
-<%block name="viewtitle">
-- - ${_('Maintenance Dashboard')} - -
- <%block name="viewtitle"> - -- ${view['name']} -
- - -<%block name="viewcontent"> -
-
-
diff --git a/cms/templates/studio_xblock_wrapper.html b/cms/templates/studio_xblock_wrapper.html
index 4fb3c71803cb..8929b225cc91 100644
--- a/cms/templates/studio_xblock_wrapper.html
+++ b/cms/templates/studio_xblock_wrapper.html
@@ -7,12 +7,11 @@
from openedx.core.djangolib.js_utils import (
dump_js_escaped_json, js_escaped_string
)
-from cms.djangoapps.contentstore.toggles import use_new_text_editor, use_new_problem_editor, use_new_video_editor, use_video_gallery_flow
+from cms.djangoapps.contentstore.toggles import use_new_problem_editor, use_new_video_editor, use_video_gallery_flow
from cms.lib.xblock.upstream_sync import UpstreamLink
from openedx.core.djangoapps.content_tagging.toggles import is_tagging_feature_disabled
%>
<%
-use_new_editor_text = use_new_text_editor(xblock.context_key)
use_new_editor_video = use_new_video_editor(xblock.context_key)
use_new_editor_problem = use_new_problem_editor(xblock.context_key)
use_new_video_gallery_flow = use_video_gallery_flow()
@@ -83,7 +82,6 @@
is-collapsed
% endif
"
- use-new-editor-text = ${use_new_editor_text}
use-new-editor-video = ${use_new_editor_video}
use-new-editor-problem = ${use_new_editor_problem}
use-video-gallery-flow = ${use_new_video_gallery_flow}
@@ -93,7 +91,7 @@
% if upstream_info.upstream_ref:
data-upstream-ref = ${upstream_info.upstream_ref}
data-version-synced = ${upstream_info.version_synced}
- data-is-modified = ${upstream_info.is_modified}
+ data-is-modified = ${len(upstream_info.downstream_customized) > 0}
%endif
>
-
- % for view in views.values():
-
- - ${view['name']} - ${view['description']} - - % endfor -
@@ -121,10 +119,10 @@
${_("Sourced from a library - but the upstream link is broken/invalid.")}
% else:
-