Feature/move mtls settings to presets

custom-recipes/api-connect/recipe.json

@@ -291,27 +291,6 @@
             "visibilityCondition": "model.auth_type!='secure_oauth' && model.auth_type!='secure_basic'",
             "defaultValue": false
         },
-        {
-            "name": "use_mtls",
-            "label": "Use mTLS",
-            "description": "",
-            "type": "BOOLEAN",
-            "defaultValue": false
-        },
-        {
-            "name": "mtls_certificate_path",
-            "label": "Path to certificate",
-            "description": "",
-            "type": "STRING",
-            "visibilityCondition": "model.use_mtls==true"
-        },
-        {
-            "name": "mtls_key_path",
-            "label": "Path to key",
-            "description": "",
-            "type": "STRING",
-            "visibilityCondition": "model.use_mtls==true"
-        },
         {
             "name": "force_csv_parameters",
             "label": "Force CSV parameters",

parameter-sets/credential/parameter-set.json

@@ -119,6 +119,27 @@
             "label": "User key/values",
             "description": "User defined keys/values that can be used later in url, query string...",
             "type": "KEY_VALUE_LIST"
+        },
+        {
+            "name": "use_mtls",
+            "label": "Use mTLS",
+            "description": "",
+            "type": "BOOLEAN",
+            "defaultValue": false
+        },
+        {
+            "name": "mtls_certificate_path",
+            "label": "Path to certificate",
+            "description": "or full certificate starting with -----BEGIN and ending with END CERTIFICATE-----",
+            "type": "PASSWORD",
+            "visibilityCondition": "model.use_mtls==true"
+        },
+        {
+            "name": "mtls_key_path",
+            "label": "Path to key",
+            "description": "or full key starting with -----BEGIN and ending with END PRIVATE KEY-----",
+            "type": "PASSWORD",
+            "visibilityCondition": "model.use_mtls==true"
         }
     ]
 }

parameter-sets/secure-basic/parameter-set.json

@@ -38,6 +38,33 @@
                     "label": "NTLM"
                 }
             ]
+        },
+        {
+            "name": "use_mtls",
+            "label": "Use mTLS",
+            "description": "",
+            "type": "BOOLEAN",
+            "defaultValue": false
+        },
+        {
+            "type": "SEPARATOR",
+            "label": "Warning",
+            "description": "Restricting access to this presset will not restrict access to the certificate and key files. This has to be done by setting the appropriate access rights on these two files.",
+            "visibilityCondition": "model.use_mtls==true"
+        },
+        {
+            "name": "mtls_certificate_path",
+            "label": "Path to certificate",
+            "description": "or full certificate starting with -----BEGIN and ending with END CERTIFICATE-----",
+            "type": "PASSWORD",
+            "visibilityCondition": "model.use_mtls==true"
+        },
+        {
+            "name": "mtls_key_path",
+            "label": "Path to key",
+            "description": "or full key starting with -----BEGIN and ending with END PRIVATE KEY-----",
+            "type": "PASSWORD",
+            "visibilityCondition": "model.use_mtls==true"
         }
     ]
 }

parameter-sets/secure-oauth/parameter-set.json

@@ -47,6 +47,33 @@
             "label": "Domain",
             "description": "",
             "type": "STRING"
+        },
+        {
+            "name": "use_mtls",
+            "label": "Use mTLS",
+            "description": "",
+            "type": "BOOLEAN",
+            "defaultValue": false
+        },
+        {
+            "type": "SEPARATOR",
+            "label": "Warning",
+            "description": "1 - Restricting access to this presset will not restrict access to the certificate and key files. This has to be done by setting the appropriate access rights on these two files.\n2 - Because the OAuth flow is not controled by the plugin, mTLS cannot be used for the retrieving the access token itself.",
+            "visibilityCondition": "model.use_mtls==true"
+        },
+        {
+            "name": "mtls_certificate_path",
+            "label": "Path to certificate",
+            "description": "or full certificate starting with -----BEGIN and ending with END CERTIFICATE-----",
+            "type": "PASSWORD",
+            "visibilityCondition": "model.use_mtls==true"
+        },
+        {
+            "name": "mtls_key_path",
+            "label": "Path to key",
+            "description": "or full key starting with -----BEGIN and ending with END PRIVATE KEY-----",
+            "type": "PASSWORD",
+            "visibilityCondition": "model.use_mtls==true"
         }
     ]
 }

plugin.json

@@ -6,8 +6,7 @@
         "description": "Retrieve data from any REST API",
         "author": "Dataiku (Alex Bourret)",
         "icon": "icon-rocket",
-        "category": "Connect",
-        "tags": ["API", "Recipe", "Dataset"],
+        "tags": ["Connector"],
         "url": "https://www.dataiku.com/product/plugins/api-connect/",
         "licenseInfo": "Apache Software License",
         "recipesCategory": "visual"

python-connectors/api-connect_dataset/connector.json

@@ -238,27 +238,6 @@
             "visibilityCondition": "model.auth_type!='secure_oauth' && model.auth_type!='secure_basic'",
             "defaultValue": false
         },
-        {
-            "name": "use_mtls",
-            "label": " ",
-            "description": "Use mTLS",
-            "type": "BOOLEAN",
-            "defaultValue": false
-        },
-        {
-            "name": "mtls_certificate_path",
-            "label": "Path to certificate",
-            "description": "",
-            "type": "STRING",
-            "visibilityCondition": "model.use_mtls==true"
-        },
-        {
-            "name": "mtls_key_path",
-            "label": "Path to key",
-            "description": "",
-            "type": "STRING",
-            "visibilityCondition": "model.use_mtls==true"
-        },
         {
             "name": "force_csv_parameters",
             "label": " ",

python-lib/dku_constants.py

@@ -1,7 +1,7 @@
 class DKUConstants(object):
     API_RESPONSE_KEY = "api_response"
-    FORBIDDEN_KEYS = ["token", "password", "api_key_value", "secure_token"]
+    FORBIDDEN_KEYS = ["token", "password", "api_key_value", "secure_token", "mtls_key_path", "mtls_certificate_path"]
     FORM_DATA_BODY_FORMAT = "FORM_DATA"
-    PLUGIN_VERSION = "1.2.7-beta.3"
+    PLUGIN_VERSION = "1.2.7-beta.5"
     RAW_BODY_FORMAT = "RAW"
     REPONSE_ERROR_KEY = "dku_error"

python-lib/rest_api_client.py

@@ -1,6 +1,7 @@
 import requests
 import time
 import copy
+import tempfile
 from pagination import Pagination
 from safe_logger import SafeLogger
 from loop_detector import LoopDetector
@@ -59,9 +60,17 @@ def __init__(self, credential, secure_credentials, endpoint, custom_key_values={
             self.requests_kwargs.update({"verify": False})
         else:
             self.requests_kwargs.update({"verify": True})
-        if endpoint.get("use_mtls", False):
-            mtls_certificate_path = endpoint.get("mtls_certificate_path")
-            mtls_key_path = endpoint.get("mtls_key_path")
+        if credential.get("use_mtls", False):
+            mtls_certificate_path = credential.get("mtls_certificate_path")
+            mtls_key_path = credential.get("mtls_key_path")
+            self.requests_kwargs.update(
+                {
+                    "cert": (mtls_certificate_path, mtls_key_path)
+                }
+            )
+        if secure_credentials.get("use_mtls", False):
+            mtls_certificate_path = secure_credentials.get("mtls_certificate_path")
+            mtls_key_path = secure_credentials.get("mtls_key_path")
             self.requests_kwargs.update(
                 {
                     "cert": (mtls_certificate_path, mtls_key_path)
@@ -176,14 +185,35 @@ def request(self, method, url, can_raise_exeption=True, **kwargs):
     def request_with_redirect_retry(self, method, url, **kwargs):
         # In case of redirection to another domain, the authorization header is not kept
         # If redirect_auth_header is true, another attempt is made with initial headers to the redirected url
-        response = self.session.request(method, url, **kwargs)
+        response = self.request_with_cert(method, url, **kwargs)
         if self.redirect_auth_header and not response.url.startswith(url):
             redirection_kwargs = copy.deepcopy(kwargs)
             redirection_kwargs.pop("params", None)  # params are contained in the redirected url
             logger.warning("Redirection ! Accessing endpoint {} with initial authorization headers".format(response.url))
-            response = self.session.request(method, response.url, **redirection_kwargs)
+            response = self.request_with_cert(method, response.url, **redirection_kwargs)
         return response
 
+    def request_with_cert(self, method, url, **kwargs):
+        cert = kwargs.get("cert", None)
+        if cert and len(cert) == 2:
+            if cert[0].startswith("-----BEGIN CERTIFICATE") and cert[1].startswith("-----BEGIN "):
+                logger.info("mTLS certificate and key are strings")
+                response = None
+                with tempfile.NamedTemporaryFile(mode="w", suffix=".crt") as tmp_certificate:
+                    with tempfile.NamedTemporaryFile(mode="w", suffix=".key") as tmp_key:
+                        tmp_certificate.write(
+                            normalize_key(cert[0])
+                        )
+                        tmp_certificate.seek(0)
+                        tmp_key.write(
+                            normalize_key(cert[1])
+                        )
+                        tmp_key.seek(0)
+                        kwargs["cert"] = (tmp_certificate.name, tmp_key.name)
+                        response = self.session.request(method, url, **kwargs)
+                return response
+        return self.session.request(method, url, **kwargs)
+
     def paginated_api_call(self, can_raise_exeption=True):
         if self.pagination.params_must_be_blanked:
             self.requests_kwargs["params"] = {}
@@ -270,3 +300,20 @@ def get_headers(response):
     if isinstance(response, requests.Response):
         return response.headers
     return None
+
+
+def normalize_key(key):
+    PROTECTED_EXPRESSIONS = [
+        "BEGIN CERTIFICATE", "END CERTIFICATE",
+        "BEGIN PRIVATE KEY", "END PRIVATE KEY",
+        "BEGIN RSA PRIVATE KEY", "END RSA PRIVATE KEY"
+    ]
+    tempo_text = str(key)
+    for expression_to_protect in PROTECTED_EXPRESSIONS:
+        protected_form = expression_to_protect.replace(" ", "")
+        tempo_text = tempo_text.replace(expression_to_protect, protected_form)
+    tempo_text = tempo_text.replace(" ", "\n")
+    for expression_to_protect in PROTECTED_EXPRESSIONS:
+        protected_form = expression_to_protect.replace(" ", "")
+        tempo_text = tempo_text.replace(protected_form, expression_to_protect)
+    return tempo_text

tests/python/integration/test_scenario.py

@@ -57,3 +57,7 @@ def test_run_api_connect_xml_handling(user_dss_clients):
 
 def test_run_api_connect_parameters_renaming(user_dss_clients):
     dss_scenario.run(user_dss_clients, project_key=TEST_PROJECT_KEY, scenario_id="COLUMNPARAMETERRENAMING")
+
+
+def test_run_api_connect_mtls(user_dss_clients):
+    dss_scenario.run(user_dss_clients, project_key=TEST_PROJECT_KEY, scenario_id="MTLS")