diff --git a/python-clusters/attach-aks-cluster/cluster.py b/python-clusters/attach-aks-cluster/cluster.py index f483452..ae741cd 100644 --- a/python-clusters/attach-aks-cluster/cluster.py +++ b/python-clusters/attach-aks-cluster/cluster.py @@ -2,10 +2,9 @@ from dataiku.cluster import Cluster from azure.mgmt.containerservice import ContainerServiceClient -from dku_utils.access import _is_none_or_blank +from dku_utils.access import _is_none_or_blank, _print_as_json from dku_utils.cluster import make_overrides, get_subscription_id from dku_azure.auth import get_credentials_from_connection_info, get_credentials_from_connection_infoV2 -from dku_azure.utils import run_and_process_cloud_error from dku_azure.utils import run_and_process_cloud_error, get_instance_metadata, get_subscription_id class MyCluster(Cluster): @@ -51,6 +50,7 @@ def start(self): def do_fetch(): return clusters_client.managed_clusters.list_cluster_admin_credentials(resource_group, cluster_name) get_credentials_result = run_and_process_cloud_error(do_fetch) + logging.info("Kubeconfig retrieved for cluster %s in %s: %s", cluster_name, resource_group, _print_as_json(get_credentials_result)) kube_config_content = get_credentials_result.kubeconfigs[0].value.decode('utf8') kube_config_path = os.path.join(os.getcwd(), 'kube_config') with open(kube_config_path, 'w') as f: @@ -58,9 +58,11 @@ def do_fetch(): overrides = make_overrides(self.config, yaml.safe_load(kube_config_content), kube_config_path) # Get other cluster infos + logging.info("Retrieving cluster information for cluster %s in %s", cluster_name, resource_group) def do_inspect(): return clusters_client.managed_clusters.get(resource_group, cluster_name) get_cluster_result = run_and_process_cloud_error(do_inspect) + logging.info("Information retrieved for cluster %s in %s: %s", cluster_name, resource_group, _print_as_json(get_cluster_result)) return [overrides, {'kube_config_path':kube_config_path, 'cluster':get_cluster_result.as_dict()}] diff --git a/python-clusters/create-aks-cluster/cluster.py b/python-clusters/create-aks-cluster/cluster.py index d2100e5..eab2f4c 100644 --- a/python-clusters/create-aks-cluster/cluster.py +++ b/python-clusters/create-aks-cluster/cluster.py @@ -9,7 +9,7 @@ from azure.core.exceptions import ResourceNotFoundError, HttpResponseError from msrestazure.azure_exceptions import CloudError -from dku_utils.access import _is_none_or_blank, _has_not_blank_property +from dku_utils.access import _is_none_or_blank, _has_not_blank_property, _print_as_json from dku_utils.cluster import make_overrides, get_cluster_from_connection_info from dku_kube.nvidia_utils import add_gpu_driver_if_needed from dku_azure.auth import get_credentials_from_connection_info, get_credentials_from_connection_infoV2 @@ -320,6 +320,7 @@ def start(self): logging.info("Start creation of cluster") def do_creation(): cluster_create_op = cluster_builder.build() + logging.info("Cluster creation results: %s", _print_as_json(cluster_create_op.__dict__)) return cluster_create_op.result() create_result = run_and_process_cloud_error(do_creation) logging.info("Cluster creation finished") @@ -380,6 +381,7 @@ def do_creation(): def do_fetch(): return clusters_client.managed_clusters.list_cluster_admin_credentials(resource_group, self.cluster_name) get_credentials_result = run_and_process_cloud_error(do_fetch) + logging.info("Kubeconfig retrieved for cluster %s in %s: %s", self.cluster_name, resource_group, _print_as_json(get_credentials_result)) kube_config_content = get_credentials_result.kubeconfigs[0].value.decode("utf8") logging.info("Writing kubeconfig file...") kube_config_path = os.path.join(os.getcwd(), "kube_config") @@ -440,6 +442,7 @@ def do_delete(): future = clusters_client.managed_clusters.begin_delete(resource_group, cluster_name) return future.result() delete_result = run_and_process_cloud_error(do_delete) + logging.info("Cluster deletion results") # delete returns void, so we poll until the cluster is really gone gone = False diff --git a/python-lib/dku_utils/access.py b/python-lib/dku_utils/access.py index 2b79e07..8250406 100644 --- a/python-lib/dku_utils/access.py +++ b/python-lib/dku_utils/access.py @@ -85,11 +85,16 @@ def _object_to_json(o): r = o.__dict__ if isinstance(r, dku_basestring_type): return r + if isinstance(r, bytearray): + return r.decode("utf-8") if isinstance(r, Iterable): if hasattr(r, 'keys'): + d = dict() for field in r.keys(): - r[field] = _object_to_json(r[field]) - return r + if not field.startswith('_'): + d[field] = _object_to_json(r[field]) + + return d else: arr = [] for entry in r: diff --git a/python-runnables/inspect-node-pools/runnable.py b/python-runnables/inspect-node-pools/runnable.py index 3939b05..b21a074 100644 --- a/python-runnables/inspect-node-pools/runnable.py +++ b/python-runnables/inspect-node-pools/runnable.py @@ -2,8 +2,6 @@ import dataiku import json, logging from dku_utils.cluster import get_cluster_from_dss_cluster -from dku_utils.access import _is_none_or_blank -from dku_azure.utils import run_and_process_cloud_error class MyRunnable(Runnable): def __init__(self, project_key, config, plugin_config): diff --git a/python-runnables/resize-node-pool/runnable.py b/python-runnables/resize-node-pool/runnable.py index a2a04b5..7b39de8 100644 --- a/python-runnables/resize-node-pool/runnable.py +++ b/python-runnables/resize-node-pool/runnable.py @@ -2,7 +2,7 @@ import dataiku import json, logging from dku_utils.cluster import get_cluster_from_dss_cluster -from dku_utils.access import _is_none_or_blank +from dku_utils.access import _is_none_or_blank, _print_as_json from dku_azure.utils import run_and_process_cloud_error class MyRunnable(Runnable): @@ -50,6 +50,7 @@ def do_update(): cluster_update_op = clusters.managed_clusters.begin_create_or_update(resource_group, cluster_name, cluster) return cluster_update_op.result() update_result = run_and_process_cloud_error(do_update) + logging.info("Cluster update results: %s", _print_as_json(update_result)) logging.info("Cluster updated") return '
%s
' % json.dumps(update_result.as_dict(), indent=2)