diff --git a/custom-recipes/api-connect/recipe.json b/custom-recipes/api-connect/recipe.json index d590d92..cc0fac3 100644 --- a/custom-recipes/api-connect/recipe.json +++ b/custom-recipes/api-connect/recipe.json @@ -291,27 +291,6 @@ "visibilityCondition": "model.auth_type!='secure_oauth' && model.auth_type!='secure_basic'", "defaultValue": false }, - { - "name": "use_mtls", - "label": "Use mTLS", - "description": "", - "type": "BOOLEAN", - "defaultValue": false - }, - { - "name": "mtls_certificate_path", - "label": "Path to certificate", - "description": "", - "type": "STRING", - "visibilityCondition": "model.use_mtls==true" - }, - { - "name": "mtls_key_path", - "label": "Path to key", - "description": "", - "type": "STRING", - "visibilityCondition": "model.use_mtls==true" - }, { "name": "force_csv_parameters", "label": "Force CSV parameters", diff --git a/parameter-sets/credential/parameter-set.json b/parameter-sets/credential/parameter-set.json index 4196159..0c027f6 100644 --- a/parameter-sets/credential/parameter-set.json +++ b/parameter-sets/credential/parameter-set.json @@ -119,6 +119,27 @@ "label": "User key/values", "description": "User defined keys/values that can be used later in url, query string...", "type": "KEY_VALUE_LIST" + }, + { + "name": "use_mtls", + "label": "Use mTLS", + "description": "", + "type": "BOOLEAN", + "defaultValue": false + }, + { + "name": "mtls_certificate_path", + "label": "Path to certificate", + "description": "or full certificate starting with -----BEGIN and ending with END CERTIFICATE-----", + "type": "PASSWORD", + "visibilityCondition": "model.use_mtls==true" + }, + { + "name": "mtls_key_path", + "label": "Path to key", + "description": "or full key starting with -----BEGIN and ending with END PRIVATE KEY-----", + "type": "PASSWORD", + "visibilityCondition": "model.use_mtls==true" } ] } diff --git a/parameter-sets/secure-basic/parameter-set.json b/parameter-sets/secure-basic/parameter-set.json index 623d24d..a6b6809 100644 --- a/parameter-sets/secure-basic/parameter-set.json +++ b/parameter-sets/secure-basic/parameter-set.json @@ -38,6 +38,33 @@ "label": "NTLM" } ] + }, + { + "name": "use_mtls", + "label": "Use mTLS", + "description": "", + "type": "BOOLEAN", + "defaultValue": false + }, + { + "type": "SEPARATOR", + "label": "Warning", + "description": "Restricting access to this presset will not restrict access to the certificate and key files. This has to be done by setting the appropriate access rights on these two files.", + "visibilityCondition": "model.use_mtls==true" + }, + { + "name": "mtls_certificate_path", + "label": "Path to certificate", + "description": "or full certificate starting with -----BEGIN and ending with END CERTIFICATE-----", + "type": "PASSWORD", + "visibilityCondition": "model.use_mtls==true" + }, + { + "name": "mtls_key_path", + "label": "Path to key", + "description": "or full key starting with -----BEGIN and ending with END PRIVATE KEY-----", + "type": "PASSWORD", + "visibilityCondition": "model.use_mtls==true" } ] } diff --git a/parameter-sets/secure-oauth/parameter-set.json b/parameter-sets/secure-oauth/parameter-set.json index 464f9ac..72c6437 100644 --- a/parameter-sets/secure-oauth/parameter-set.json +++ b/parameter-sets/secure-oauth/parameter-set.json @@ -47,6 +47,33 @@ "label": "Domain", "description": "", "type": "STRING" + }, + { + "name": "use_mtls", + "label": "Use mTLS", + "description": "", + "type": "BOOLEAN", + "defaultValue": false + }, + { + "type": "SEPARATOR", + "label": "Warning", + "description": "1 - Restricting access to this presset will not restrict access to the certificate and key files. This has to be done by setting the appropriate access rights on these two files.\n2 - Because the OAuth flow is not controled by the plugin, mTLS cannot be used for the retrieving the access token itself.", + "visibilityCondition": "model.use_mtls==true" + }, + { + "name": "mtls_certificate_path", + "label": "Path to certificate", + "description": "or full certificate starting with -----BEGIN and ending with END CERTIFICATE-----", + "type": "PASSWORD", + "visibilityCondition": "model.use_mtls==true" + }, + { + "name": "mtls_key_path", + "label": "Path to key", + "description": "or full key starting with -----BEGIN and ending with END PRIVATE KEY-----", + "type": "PASSWORD", + "visibilityCondition": "model.use_mtls==true" } ] } diff --git a/plugin.json b/plugin.json index afb4273..88ad213 100644 --- a/plugin.json +++ b/plugin.json @@ -6,8 +6,7 @@ "description": "Retrieve data from any REST API", "author": "Dataiku (Alex Bourret)", "icon": "icon-rocket", - "category": "Connect", - "tags": ["API", "Recipe", "Dataset"], + "tags": ["Connector"], "url": "https://www.dataiku.com/product/plugins/api-connect/", "licenseInfo": "Apache Software License", "recipesCategory": "visual" diff --git a/python-connectors/api-connect_dataset/connector.json b/python-connectors/api-connect_dataset/connector.json index 0dac288..2f34370 100644 --- a/python-connectors/api-connect_dataset/connector.json +++ b/python-connectors/api-connect_dataset/connector.json @@ -238,27 +238,6 @@ "visibilityCondition": "model.auth_type!='secure_oauth' && model.auth_type!='secure_basic'", "defaultValue": false }, - { - "name": "use_mtls", - "label": " ", - "description": "Use mTLS", - "type": "BOOLEAN", - "defaultValue": false - }, - { - "name": "mtls_certificate_path", - "label": "Path to certificate", - "description": "", - "type": "STRING", - "visibilityCondition": "model.use_mtls==true" - }, - { - "name": "mtls_key_path", - "label": "Path to key", - "description": "", - "type": "STRING", - "visibilityCondition": "model.use_mtls==true" - }, { "name": "force_csv_parameters", "label": " ", diff --git a/python-lib/dku_constants.py b/python-lib/dku_constants.py index 8bf962b..7e007f8 100644 --- a/python-lib/dku_constants.py +++ b/python-lib/dku_constants.py @@ -1,7 +1,7 @@ class DKUConstants(object): API_RESPONSE_KEY = "api_response" - FORBIDDEN_KEYS = ["token", "password", "api_key_value", "secure_token"] + FORBIDDEN_KEYS = ["token", "password", "api_key_value", "secure_token", "mtls_key_path", "mtls_certificate_path"] FORM_DATA_BODY_FORMAT = "FORM_DATA" - PLUGIN_VERSION = "1.2.7-beta.3" + PLUGIN_VERSION = "1.2.7-beta.5" RAW_BODY_FORMAT = "RAW" REPONSE_ERROR_KEY = "dku_error" diff --git a/python-lib/rest_api_client.py b/python-lib/rest_api_client.py index 57a542e..cb4e824 100644 --- a/python-lib/rest_api_client.py +++ b/python-lib/rest_api_client.py @@ -1,6 +1,7 @@ import requests import time import copy +import tempfile from pagination import Pagination from safe_logger import SafeLogger from loop_detector import LoopDetector @@ -59,9 +60,17 @@ def __init__(self, credential, secure_credentials, endpoint, custom_key_values={ self.requests_kwargs.update({"verify": False}) else: self.requests_kwargs.update({"verify": True}) - if endpoint.get("use_mtls", False): - mtls_certificate_path = endpoint.get("mtls_certificate_path") - mtls_key_path = endpoint.get("mtls_key_path") + if credential.get("use_mtls", False): + mtls_certificate_path = credential.get("mtls_certificate_path") + mtls_key_path = credential.get("mtls_key_path") + self.requests_kwargs.update( + { + "cert": (mtls_certificate_path, mtls_key_path) + } + ) + if secure_credentials.get("use_mtls", False): + mtls_certificate_path = secure_credentials.get("mtls_certificate_path") + mtls_key_path = secure_credentials.get("mtls_key_path") self.requests_kwargs.update( { "cert": (mtls_certificate_path, mtls_key_path) @@ -176,14 +185,35 @@ def request(self, method, url, can_raise_exeption=True, **kwargs): def request_with_redirect_retry(self, method, url, **kwargs): # In case of redirection to another domain, the authorization header is not kept # If redirect_auth_header is true, another attempt is made with initial headers to the redirected url - response = self.session.request(method, url, **kwargs) + response = self.request_with_cert(method, url, **kwargs) if self.redirect_auth_header and not response.url.startswith(url): redirection_kwargs = copy.deepcopy(kwargs) redirection_kwargs.pop("params", None) # params are contained in the redirected url logger.warning("Redirection ! Accessing endpoint {} with initial authorization headers".format(response.url)) - response = self.session.request(method, response.url, **redirection_kwargs) + response = self.request_with_cert(method, response.url, **redirection_kwargs) return response + def request_with_cert(self, method, url, **kwargs): + cert = kwargs.get("cert", None) + if cert and len(cert) == 2: + if cert[0].startswith("-----BEGIN CERTIFICATE") and cert[1].startswith("-----BEGIN "): + logger.info("mTLS certificate and key are strings") + response = None + with tempfile.NamedTemporaryFile(mode="w", suffix=".crt") as tmp_certificate: + with tempfile.NamedTemporaryFile(mode="w", suffix=".key") as tmp_key: + tmp_certificate.write( + normalize_key(cert[0]) + ) + tmp_certificate.seek(0) + tmp_key.write( + normalize_key(cert[1]) + ) + tmp_key.seek(0) + kwargs["cert"] = (tmp_certificate.name, tmp_key.name) + response = self.session.request(method, url, **kwargs) + return response + return self.session.request(method, url, **kwargs) + def paginated_api_call(self, can_raise_exeption=True): if self.pagination.params_must_be_blanked: self.requests_kwargs["params"] = {} @@ -270,3 +300,20 @@ def get_headers(response): if isinstance(response, requests.Response): return response.headers return None + + +def normalize_key(key): + PROTECTED_EXPRESSIONS = [ + "BEGIN CERTIFICATE", "END CERTIFICATE", + "BEGIN PRIVATE KEY", "END PRIVATE KEY", + "BEGIN RSA PRIVATE KEY", "END RSA PRIVATE KEY" + ] + tempo_text = str(key) + for expression_to_protect in PROTECTED_EXPRESSIONS: + protected_form = expression_to_protect.replace(" ", "") + tempo_text = tempo_text.replace(expression_to_protect, protected_form) + tempo_text = tempo_text.replace(" ", "\n") + for expression_to_protect in PROTECTED_EXPRESSIONS: + protected_form = expression_to_protect.replace(" ", "") + tempo_text = tempo_text.replace(protected_form, expression_to_protect) + return tempo_text diff --git a/tests/python/integration/test_scenario.py b/tests/python/integration/test_scenario.py index 63c41af..315b6fc 100644 --- a/tests/python/integration/test_scenario.py +++ b/tests/python/integration/test_scenario.py @@ -57,3 +57,7 @@ def test_run_api_connect_xml_handling(user_dss_clients): def test_run_api_connect_parameters_renaming(user_dss_clients): dss_scenario.run(user_dss_clients, project_key=TEST_PROJECT_KEY, scenario_id="COLUMNPARAMETERRENAMING") + + +def test_run_api_connect_mtls(user_dss_clients): + dss_scenario.run(user_dss_clients, project_key=TEST_PROJECT_KEY, scenario_id="MTLS")