Af tree explorer recipe

.gitignore

@@ -233,3 +233,6 @@ gradle-app.setting
 setup.cfg
 tests/allure_report
 prototypes
+
+# Code editor configuration
+.idea/

CHANGELOG.md

@@ -1,5 +1,9 @@
 # Changelog
 
+## [Version 1.4.2](https://github.com/dataiku/dss-plugin-pi-server/releases/tag/v1.4.2) - Feature release - 2025-11-26
+
+- Add a AF hierarchy downloader
+
 ## [Version 1.4.0](https://github.com/dataiku/dss-plugin-pi-server/releases/tag/v1.3.1) - Bugfix release - 2025-05-24
 
 - Add write recipe

custom-recipes/pi-system-af-tree/recipe.json

@@ -0,0 +1,115 @@
+{
+    "meta": {
+        "label": "AF tree explorer",
+        "description": "Explore the AF tree",
+        "icon": "icon-pi-system icon-cogs"
+    },
+    "kind": "PYTHON",
+    "paramsPythonSetup": "browse_af_tree.py",
+    "inputRoles": [
+    ],
+
+    "outputRoles": [
+        {
+            "name": "api_output",
+            "label": "Attributes dataset",
+            "description": "",
+            "arity": "UNARY",
+            "required": true,
+            "acceptsDataset": true
+        }
+    ],
+    "paramsTemplate" : "pi-system_af-explorer.html",
+    "paramsModule" : "piSystemTreeApp.module",
+    "params": [
+        {
+            "type": "SEPARATOR",
+            "label": "Authentication"
+        },
+        {
+            "name": "credentials",
+            "label": "User preset",
+            "type": "PRESET",
+            "parameterSetId": "basic-auth"
+        },
+        {
+            "name": "show_advanced_parameters",
+            "label": "Show advanced parameters",
+            "type": "BOOLEAN",
+            "definition": "",
+            "defaultValue": false
+        },
+        {
+            "name": "use_server_url_column",
+            "label": "Use server value per row",
+            "visibilityCondition": "model.show_advanced_parameters==true && false",
+            "description": "",
+            "type": "BOOLEAN",
+            "defaultValue": false
+        },
+        {
+            "visibilityCondition": "(model.use_server_url_column==true) && (model.show_advanced_parameters==true)",
+            "name": "server_url_column",
+            "label": "Server domain columnn",
+            "description": "Should match the required path for each row",
+            "type": "COLUMN",
+            "columnRole": "input_dataset"
+        },
+        {
+            "visibilityCondition": "(model.use_server_url_column==false) && (model.show_advanced_parameters==true)",
+            "name": "server_url",
+            "label": "Server URL",
+            "type": "STRING",
+            "definition": "https://my_server:8082",
+            "defaultValue": ""
+        },
+        {
+            "name": "is_ssl_check_disabled",
+            "label": "Disable SSL check",
+            "visibilityCondition": "model.show_advanced_parameters==true",
+            "type": "BOOLEAN",
+            "definition": "",
+            "defaultValue": true
+        },
+        {
+            "name": "ssl_cert_path",
+            "label": "Path to SSL certificate",
+            "type": "STRING",
+            "description": "(optional)",
+            "visibilityCondition": "model.show_advanced_parameters==true && model.is_ssl_check_disabled==false",
+            "mandatory": false
+        },
+        {
+            "name": "must_convert_object_to_string",
+            "label": "Convert objects to string",
+            "visibilityCondition": "model.show_advanced_parameters==true",
+            "type": "BOOLEAN",
+            "description": "(for direct output to databases)",
+            "defaultValue": false
+        },
+        {
+            "name": "is_debug_mode",
+            "label": "Verbose logging",
+            "visibilityCondition": "model.show_advanced_parameters==true",
+            "type": "BOOLEAN",
+            "description": "",
+            "defaultValue": false
+        },
+        {
+            "name": "server_name",
+            "label": "Server name",
+            "type": "SELECT",
+            "description": "",
+            "getChoicesFromPython": true
+        },
+        {
+            "name": "database_name",
+            "label": "Database name",
+            "type": "SELECT",
+            "description": "",
+            "visibilityCondition": "model.server_name.length>=0",
+            "getChoicesFromPython": true
+        }
+    ],
+    "resourceKeys": []
+}

custom-recipes/pi-system-af-tree/recipe.py

@@ -0,0 +1,77 @@
+import dataiku
+from dataiku.customrecipe import get_recipe_config, get_output_names_for_role
+from safe_logger import SafeLogger
+from osisoft_plugin_common import (
+    get_credentials, PerformanceTimer
+)
+from osisoft_constants import OSIsoftConstants
+
+
+logger = SafeLogger("pi-system plugin", forbiden_keys=["token", "password"])
+
+logger.info("PIWebAPI AF selector recipe v{}".format(
+    OSIsoftConstants.PLUGIN_VERSION
+))
+
+
+def get_step_value(item):
+    if item and "Step" in item:
+        if item.get("Step") is True:
+            return "True"
+        else:
+            return "False"
+    return None
+
+
+def next_tree_item(tree_data):
+    if not isinstance(tree_data, list):
+        return
+    for item in tree_data:
+        children = item.pop("children", [])
+        if children:
+            for child in next_tree_item(children):
+                yield child
+        yield item
+
+
+output_names_stats = get_output_names_for_role('api_output')
+config = get_recipe_config()
+tree_data = config.get("treeData", [])
+
+logger.info("Initialization with config config={}".format(logger.filter_secrets(config)))
+
+auth_type, username, password, server_url, is_ssl_check_disabled = get_credentials(config)
+is_ssl_check_disabled = config.get("is_ssl_check_disabled", False)  # Because no advanced parameter switch
+
+network_timer = PerformanceTimer()
+processing_timer = PerformanceTimer()
+processing_timer.start()
+
+output_dataset = dataiku.Dataset(output_names_stats[0])
+schema = [
+    {'name': 'title', 'type': 'string'},
+    {'name': 'template_name', 'type': 'string'},
+    {'name': 'category_names', 'type': 'string'},
+    {'name': 'path', 'type': 'string'},
+    {'name': 'paths', 'type': 'string'},
+    {'name': 'id', 'type': 'string'},
+    {'name': 'url', 'type': 'string'},
+    {'name': 'data_type', 'type': 'string'},
+    {'name': 'summary_type', 'type': 'array'},
+    {'name': 'boundary_type', 'type': 'string'},
+    {'name': 'record_boundary_type', 'type': 'string'},
+    {'name': 'summary_duration', 'type': 'string'},
+    {'name': 'max_count', 'type': 'int'},
+
+]
+output_dataset.write_schema(schema)
+
+selectedAttributes = config.get("outputSelectedAttributes", [])
+with output_dataset.get_writer() as writer:
+    for item in selectedAttributes:
+        if item.get("checked", True) is True:
+            writer.write_row_dict(item)
+
+processing_timer.stop()
+logger.info("Overall timer:{}".format(processing_timer.get_report()))
+logger.info("Network timer:{}".format(network_timer.get_report()))

custom-recipes/pi-system-retrieve-list/recipe.json

@@ -105,7 +105,7 @@
             "name": "use_batch_mode",
             "label": "Use batch mode",
             "type": "BOOLEAN",
-            "description": "",
+            "description": "Use to quickly retrieve small samples from multiple paths. ⚠️Not for large time ranges",
             "visibilityCondition": "model.show_advanced_parameters==true",
             "defaultValue": false
         },

custom-recipes/pi-system-retrieve-list/recipe.py

@@ -7,7 +7,8 @@
     get_credentials, get_interpolated_parameters, normalize_af_path,
     get_combined_description, get_base_for_data_type, check_debug_mode,
     PerformanceTimer, get_max_count, check_must_convert_object_to_string,
-    convert_schema_objects_to_string, get_summary_parameters, get_advanced_parameters
+    convert_schema_objects_to_string, get_summary_parameters, get_advanced_parameters,
+    get_batch_parameters
 )
 from osisoft_client import OSIsoftClient
 from osisoft_constants import OSIsoftConstants
@@ -63,6 +64,8 @@ def get_step_value(item):
 record_boundary_type = config.get("record_boundary_type") if data_type == "RecordedData" else None
 summary_type, summary_duration = get_summary_parameters(config)
 do_duplicate_input_row = config.get("do_duplicate_input_row", False)
+max_request_size, estimated_density, maximum_points_returned = get_batch_parameters(config)
+max_time_to_retrieve_per_batch = estimated_density / maximum_points_returned #density per hour <- max time is in hour
 
 network_timer = PerformanceTimer()
 processing_timer = PerformanceTimer()
@@ -150,7 +153,9 @@ def get_step_value(item):
                     object_id=object_id,
                     summary_type=summary_type,
                     summary_duration=summary_duration,
-                    endpoint_type="AF"
+                    endpoint_type="AF",
+                    estimated_density=estimated_density,
+                    maximum_points_returned=maximum_points_returned
                 )
                 batch_buffer_size = 0
                 buffer = []