From 8623295e787286d32b7aca1dd56a6371b06efb1d Mon Sep 17 00:00:00 2001 From: osuleiman <34806809+osuleiman@users.noreply.github.com> Date: Wed, 17 May 2023 12:16:12 +0300 Subject: [PATCH 01/11] Update types.py Added new 'AUTO' format. --- databricks_cli/workspace/types.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/databricks_cli/workspace/types.py b/databricks_cli/workspace/types.py index 8e685eac..8670cfa8 100644 --- a/databricks_cli/workspace/types.py +++ b/databricks_cli/workspace/types.py @@ -50,6 +50,8 @@ def to_language_and_format(cls, path): language_and_format = (None, WorkspaceFormat.HTML) elif ext == '.dbc': language_and_format = (None, WorkspaceFormat.DBC) + else: + language_and_format = (None, WorkspaceFormat.AUTO) return language_and_format @classmethod @@ -87,7 +89,8 @@ class WorkspaceFormat(object): HTML = 'HTML' JUPYTER = 'JUPYTER' DBC = 'DBC' - ALL = [SOURCE, HTML, JUPYTER, DBC] + AUTO = 'AUTO' + ALL = [SOURCE, HTML, JUPYTER, DBC, AUTO] class FormatClickType(ParamType): From b5c5917c4ce55894bd58aa337e5a20261a8579ff Mon Sep 17 00:00:00 2001 From: osuleiman <34806809+osuleiman@users.noreply.github.com> Date: Wed, 17 May 2023 12:18:29 +0300 Subject: [PATCH 02/11] Update cli.py --- databricks_cli/workspace/cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/databricks_cli/workspace/cli.py b/databricks_cli/workspace/cli.py index be03c0a8..ca3b66ca 100644 --- a/databricks_cli/workspace/cli.py +++ b/databricks_cli/workspace/cli.py @@ -95,7 +95,7 @@ def import_workspace_cli(api_client, source_path, target_path, language, format, """ Imports a file from local to the Databricks workspace. - The format is by default SOURCE. Possible formats are SOURCE, HTML, JUPYTER, and DBC. Each + The format is by default SOURCE. Possible formats are SOURCE, HTML, JUPYTER, DBC, and AUTO. Each format is documented at https://docs.databricks.com/api/latest/workspace.html#notebookexportformat. """ From 29f8c39c5dd8023ba6458968104358e7942c6c3d Mon Sep 17 00:00:00 2001 From: osuleiman <34806809+osuleiman@users.noreply.github.com> Date: Wed, 17 May 2023 12:26:39 +0300 Subject: [PATCH 03/11] Update api.py --- databricks_cli/workspace/api.py | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/databricks_cli/workspace/api.py b/databricks_cli/workspace/api.py index e38db7bf..eb56d623 100644 --- a/databricks_cli/workspace/api.py +++ b/databricks_cli/workspace/api.py @@ -149,16 +149,11 @@ def import_workspace_dir(self, source_path, target_path, overwrite, exclude_hidd headers=headers) elif os.path.isfile(cur_src): ext = WorkspaceLanguage.get_extension(cur_src) - if ext != '': - cur_dst = cur_dst[:-len(ext)] - (language, file_format) = WorkspaceLanguage.to_language_and_format(cur_src) - self.import_workspace(cur_src, cur_dst, language, file_format, overwrite, - headers=headers) - click.echo('{} -> {}'.format(cur_src, cur_dst)) - else: - extensions = ', '.join(WorkspaceLanguage.EXTENSIONS) - click.echo(('{} does not have a valid extension of {}. Skip this file and ' + - 'continue.').format(cur_src, extensions)) + cur_dst = cur_dst[:-len(ext)] + (language, file_format) = WorkspaceLanguage.to_language_and_format(cur_src) + self.import_workspace(cur_src, cur_dst, language, file_format, overwrite, + headers=headers) + click.echo('{} -> {}'.format(cur_src, cur_dst)) def export_workspace_dir(self, source_path, target_path, overwrite, headers=None): if os.path.isfile(target_path): From 76c772c811744bc91998a0ad5837938e04db170b Mon Sep 17 00:00:00 2001 From: osuleiman <34806809+osuleiman@users.noreply.github.com> Date: Wed, 17 May 2023 12:37:14 +0300 Subject: [PATCH 04/11] Update cli.py --- databricks_cli/workspace/cli.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/databricks_cli/workspace/cli.py b/databricks_cli/workspace/cli.py index ca3b66ca..24966775 100644 --- a/databricks_cli/workspace/cli.py +++ b/databricks_cli/workspace/cli.py @@ -83,7 +83,8 @@ def mkdirs_cli(api_client, workspace_path): short_help='Imports a file from local to the Databricks workspace.') @click.argument('source_path') @click.argument('target_path') -@click.option('--language', '-l', required=True, type=LanguageClickType(), +@click.option('-- + ', '-l', required=False, type=LanguageClickType(), help=', '.join(WorkspaceLanguage.ALL)) @click.option('--format', '-f', default=WorkspaceFormat.SOURCE, type=FormatClickType()) @click.option('--overwrite', '-o', is_flag=True, default=False) @@ -91,7 +92,7 @@ def mkdirs_cli(api_client, workspace_path): @profile_option @eat_exceptions @provide_api_client -def import_workspace_cli(api_client, source_path, target_path, language, format, overwrite): # NOQA +def import_workspace_cli(api_client, source_path, target_path, language=None, format, overwrite): # NOQA """ Imports a file from local to the Databricks workspace. From 2d719c92a62b801ad62366e96d740a74e4a01117 Mon Sep 17 00:00:00 2001 From: osuleiman <34806809+osuleiman@users.noreply.github.com> Date: Wed, 17 May 2023 12:38:50 +0300 Subject: [PATCH 05/11] Update cli.py --- databricks_cli/workspace/cli.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/databricks_cli/workspace/cli.py b/databricks_cli/workspace/cli.py index 24966775..4060fe12 100644 --- a/databricks_cli/workspace/cli.py +++ b/databricks_cli/workspace/cli.py @@ -83,8 +83,7 @@ def mkdirs_cli(api_client, workspace_path): short_help='Imports a file from local to the Databricks workspace.') @click.argument('source_path') @click.argument('target_path') -@click.option('-- - ', '-l', required=False, type=LanguageClickType(), +@click.option('--language', '-l', required=True, type=LanguageClickType(), help=', '.join(WorkspaceLanguage.ALL)) @click.option('--format', '-f', default=WorkspaceFormat.SOURCE, type=FormatClickType()) @click.option('--overwrite', '-o', is_flag=True, default=False) From 43e4aac306e77d06ee71c57cdf2c1e59535b919f Mon Sep 17 00:00:00 2001 From: osuleiman <34806809+osuleiman@users.noreply.github.com> Date: Wed, 17 May 2023 12:41:18 +0300 Subject: [PATCH 06/11] Update cli.py --- databricks_cli/workspace/cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/databricks_cli/workspace/cli.py b/databricks_cli/workspace/cli.py index 4060fe12..104e500f 100644 --- a/databricks_cli/workspace/cli.py +++ b/databricks_cli/workspace/cli.py @@ -91,7 +91,7 @@ def mkdirs_cli(api_client, workspace_path): @profile_option @eat_exceptions @provide_api_client -def import_workspace_cli(api_client, source_path, target_path, language=None, format, overwrite): # NOQA +def import_workspace_cli(api_client, source_path, target_path, format, overwrite, language=None): # NOQA """ Imports a file from local to the Databricks workspace. From 7f9264c880db02fe50b4c50850017ef87cb17826 Mon Sep 17 00:00:00 2001 From: osuleiman <34806809+osuleiman@users.noreply.github.com> Date: Wed, 17 May 2023 12:42:58 +0300 Subject: [PATCH 07/11] Update cli.py --- databricks_cli/workspace/cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/databricks_cli/workspace/cli.py b/databricks_cli/workspace/cli.py index 104e500f..80ca3d6d 100644 --- a/databricks_cli/workspace/cli.py +++ b/databricks_cli/workspace/cli.py @@ -83,7 +83,7 @@ def mkdirs_cli(api_client, workspace_path): short_help='Imports a file from local to the Databricks workspace.') @click.argument('source_path') @click.argument('target_path') -@click.option('--language', '-l', required=True, type=LanguageClickType(), +@click.option('--language', '-l', required=False, type=LanguageClickType(), help=', '.join(WorkspaceLanguage.ALL)) @click.option('--format', '-f', default=WorkspaceFormat.SOURCE, type=FormatClickType()) @click.option('--overwrite', '-o', is_flag=True, default=False) From 93c12eba79da26ae49005f96b783a7c87c1e6282 Mon Sep 17 00:00:00 2001 From: osuleiman <34806809+osuleiman@users.noreply.github.com> Date: Wed, 17 May 2023 14:01:21 +0300 Subject: [PATCH 08/11] Update api.py --- databricks_cli/workspace/api.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/databricks_cli/workspace/api.py b/databricks_cli/workspace/api.py index eb56d623..569645a5 100644 --- a/databricks_cli/workspace/api.py +++ b/databricks_cli/workspace/api.py @@ -132,7 +132,7 @@ def export_workspace(self, source_path, target_path, fmt, is_overwrite, headers= def delete(self, workspace_path, is_recursive, headers=None): self.client.delete(workspace_path, is_recursive, headers=headers) - def import_workspace_dir(self, source_path, target_path, overwrite, exclude_hidden_files, + def import_workspace_dir(self, source_path, target_path, overwrite, exclude_hidden_files, are_workspace_files=False headers=None): # pylint: disable=too-many-locals filenames = os.listdir(source_path) @@ -149,7 +149,8 @@ def import_workspace_dir(self, source_path, target_path, overwrite, exclude_hidd headers=headers) elif os.path.isfile(cur_src): ext = WorkspaceLanguage.get_extension(cur_src) - cur_dst = cur_dst[:-len(ext)] + if not are_workspace_files: + cur_dst = cur_dst[:-len(ext)] (language, file_format) = WorkspaceLanguage.to_language_and_format(cur_src) self.import_workspace(cur_src, cur_dst, language, file_format, overwrite, headers=headers) From 670c4f67c6831a7757fb70b25d4ff84c078ac566 Mon Sep 17 00:00:00 2001 From: osuleiman <34806809+osuleiman@users.noreply.github.com> Date: Wed, 17 May 2023 14:05:14 +0300 Subject: [PATCH 09/11] Update cli.py --- databricks_cli/workspace/cli.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/databricks_cli/workspace/cli.py b/databricks_cli/workspace/cli.py index 80ca3d6d..edd41032 100644 --- a/databricks_cli/workspace/cli.py +++ b/databricks_cli/workspace/cli.py @@ -87,6 +87,7 @@ def mkdirs_cli(api_client, workspace_path): help=', '.join(WorkspaceLanguage.ALL)) @click.option('--format', '-f', default=WorkspaceFormat.SOURCE, type=FormatClickType()) @click.option('--overwrite', '-o', is_flag=True, default=False) +@click.option('--are_workspace_files', '-wf', is_flag=True, default=False) @debug_option @profile_option @eat_exceptions @@ -180,15 +181,16 @@ def export_dir_cli(api_client, source_path, target_path, overwrite): @profile_option @eat_exceptions @provide_api_client -def import_dir_cli(api_client, source_path, target_path, overwrite, exclude_hidden_files): +def import_dir_cli(api_client, source_path, target_path, overwrite, exclude_hidden_files, are_workspace_files): """ Recursively imports a directory from local to the Databricks workspace. - Only directories and files with the extensions .scala, .py, .sql, .r, .R, .ipynb are imported. + Only directories and files with the extensions .scala, .py, .sql, .r, .R, .ipynb are imported by default. + If are_workspace_files is set to True When imported, these extensions will be stripped off the name of the notebook. """ WorkspaceApi(api_client).import_workspace_dir(source_path, target_path, overwrite, - exclude_hidden_files) + exclude_hidden_files, are_workspace_files) @click.group(context_settings=CONTEXT_SETTINGS, From c439a129be936178eba7813d06fb4b3b56fd842f Mon Sep 17 00:00:00 2001 From: osuleiman <34806809+osuleiman@users.noreply.github.com> Date: Wed, 17 May 2023 14:06:51 +0300 Subject: [PATCH 10/11] Update api.py --- databricks_cli/workspace/api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/databricks_cli/workspace/api.py b/databricks_cli/workspace/api.py index 569645a5..28396d7b 100644 --- a/databricks_cli/workspace/api.py +++ b/databricks_cli/workspace/api.py @@ -132,7 +132,7 @@ def export_workspace(self, source_path, target_path, fmt, is_overwrite, headers= def delete(self, workspace_path, is_recursive, headers=None): self.client.delete(workspace_path, is_recursive, headers=headers) - def import_workspace_dir(self, source_path, target_path, overwrite, exclude_hidden_files, are_workspace_files=False + def import_workspace_dir(self, source_path, target_path, overwrite, exclude_hidden_files, are_workspace_files, headers=None): # pylint: disable=too-many-locals filenames = os.listdir(source_path) From 18ed54718e157eed8c6219e6fa9c1b3b40c860ca Mon Sep 17 00:00:00 2001 From: osuleiman <34806809+osuleiman@users.noreply.github.com> Date: Wed, 17 May 2023 14:11:28 +0300 Subject: [PATCH 11/11] Update cli.py --- databricks_cli/workspace/cli.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/databricks_cli/workspace/cli.py b/databricks_cli/workspace/cli.py index edd41032..b51f914a 100644 --- a/databricks_cli/workspace/cli.py +++ b/databricks_cli/workspace/cli.py @@ -87,7 +87,6 @@ def mkdirs_cli(api_client, workspace_path): help=', '.join(WorkspaceLanguage.ALL)) @click.option('--format', '-f', default=WorkspaceFormat.SOURCE, type=FormatClickType()) @click.option('--overwrite', '-o', is_flag=True, default=False) -@click.option('--are_workspace_files', '-wf', is_flag=True, default=False) @debug_option @profile_option @eat_exceptions @@ -177,6 +176,7 @@ def export_dir_cli(api_client, source_path, target_path, overwrite): @click.argument('target_path') @click.option('--overwrite', '-o', is_flag=True, default=False) @click.option('--exclude-hidden-files', '-e', is_flag=True, default=False) +@click.option('--are_workspace_files', '-wf', is_flag=True, default=False) @debug_option @profile_option @eat_exceptions @@ -185,9 +185,9 @@ def import_dir_cli(api_client, source_path, target_path, overwrite, exclude_hidd """ Recursively imports a directory from local to the Databricks workspace. - Only directories and files with the extensions .scala, .py, .sql, .r, .R, .ipynb are imported by default. - If are_workspace_files is set to True + Only directories and files with the extensions .scala, .py, .sql, .r, .R, .ipynb are imported by default and are converted to notebooks compatible with the original extension. When imported, these extensions will be stripped off the name of the notebook. + If are_workspace_files is set to True, the files will be imported as as with their original extensions. """ WorkspaceApi(api_client).import_workspace_dir(source_path, target_path, overwrite, exclude_hidden_files, are_workspace_files)