diff --git a/.gitignore b/.gitignore index 7afb1838..dfe72304 100644 --- a/.gitignore +++ b/.gitignore @@ -233,3 +233,6 @@ gradle-app.setting setup.cfg tests/allure_report prototypes + +# Code editor configuration +.idea/ diff --git a/CHANGELOG.md b/CHANGELOG.md index bee789b8..d535653f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ # Changelog +## [Version 1.4.2](https://github.com/dataiku/dss-plugin-pi-server/releases/tag/v1.4.2) - Feature release - 2025-11-26 + +- Add a AF hierarchy downloader + ## [Version 1.4.0](https://github.com/dataiku/dss-plugin-pi-server/releases/tag/v1.3.1) - Bugfix release - 2025-05-24 - Add write recipe diff --git a/custom-recipes/pi-system-af-tree/recipe.json b/custom-recipes/pi-system-af-tree/recipe.json new file mode 100644 index 00000000..05439d53 --- /dev/null +++ b/custom-recipes/pi-system-af-tree/recipe.json @@ -0,0 +1,115 @@ +{ + "meta": { + "label": "AF tree explorer", + "description": "Explore the AF tree", + "icon": "icon-pi-system icon-cogs" + }, + "kind": "PYTHON", + "paramsPythonSetup": "browse_af_tree.py", + "inputRoles": [ + ], + + "outputRoles": [ + { + "name": "api_output", + "label": "Attributes dataset", + "description": "", + "arity": "UNARY", + "required": true, + "acceptsDataset": true + } + ], + "paramsTemplate" : "pi-system_af-explorer.html", + "paramsModule" : "piSystemTreeApp.module", + "params": [ + { + "type": "SEPARATOR", + "label": "Authentication" + }, + { + "name": "credentials", + "label": "User preset", + "type": "PRESET", + "parameterSetId": "basic-auth" + }, + { + "name": "show_advanced_parameters", + "label": "Show advanced parameters", + "type": "BOOLEAN", + "definition": "", + "defaultValue": false + }, + { + "name": "use_server_url_column", + "label": "Use server value per row", + "visibilityCondition": "model.show_advanced_parameters==true && false", + "description": "", + "type": "BOOLEAN", + "defaultValue": false + }, + { + "visibilityCondition": "(model.use_server_url_column==true) && (model.show_advanced_parameters==true)", + "name": "server_url_column", + "label": "Server domain columnn", + "description": "Should match the required path for each row", + "type": "COLUMN", + "columnRole": "input_dataset" + }, + { + "visibilityCondition": "(model.use_server_url_column==false) && (model.show_advanced_parameters==true)", + "name": "server_url", + "label": "Server URL", + "type": "STRING", + "definition": "https://my_server:8082", + "defaultValue": "" + }, + { + "name": "is_ssl_check_disabled", + "label": "Disable SSL check", + "visibilityCondition": "model.show_advanced_parameters==true", + "type": "BOOLEAN", + "definition": "", + "defaultValue": true + }, + { + "name": "ssl_cert_path", + "label": "Path to SSL certificate", + "type": "STRING", + "description": "(optional)", + "visibilityCondition": "model.show_advanced_parameters==true && model.is_ssl_check_disabled==false", + "mandatory": false + }, + { + "name": "must_convert_object_to_string", + "label": "Convert objects to string", + "visibilityCondition": "model.show_advanced_parameters==true", + "type": "BOOLEAN", + "description": "(for direct output to databases)", + "defaultValue": false + }, + { + "name": "is_debug_mode", + "label": "Verbose logging", + "visibilityCondition": "model.show_advanced_parameters==true", + "type": "BOOLEAN", + "description": "", + "defaultValue": false + }, + { + "name": "server_name", + "label": "Server name", + "type": "SELECT", + "description": "", + "getChoicesFromPython": true + }, + { + "name": "database_name", + "label": "Database name", + "type": "SELECT", + "description": "", + "visibilityCondition": "model.server_name.length>=0", + "getChoicesFromPython": true + } + ], + "resourceKeys": [] +} diff --git a/custom-recipes/pi-system-af-tree/recipe.py b/custom-recipes/pi-system-af-tree/recipe.py new file mode 100644 index 00000000..101ec8c6 --- /dev/null +++ b/custom-recipes/pi-system-af-tree/recipe.py @@ -0,0 +1,77 @@ +import dataiku +from dataiku.customrecipe import get_recipe_config, get_output_names_for_role +from safe_logger import SafeLogger +from osisoft_plugin_common import ( + get_credentials, PerformanceTimer +) +from osisoft_constants import OSIsoftConstants + + +logger = SafeLogger("pi-system plugin", forbiden_keys=["token", "password"]) + +logger.info("PIWebAPI AF selector recipe v{}".format( + OSIsoftConstants.PLUGIN_VERSION +)) + + +def get_step_value(item): + if item and "Step" in item: + if item.get("Step") is True: + return "True" + else: + return "False" + return None + + +def next_tree_item(tree_data): + if not isinstance(tree_data, list): + return + for item in tree_data: + children = item.pop("children", []) + if children: + for child in next_tree_item(children): + yield child + yield item + + +output_names_stats = get_output_names_for_role('api_output') +config = get_recipe_config() +tree_data = config.get("treeData", []) + +logger.info("Initialization with config config={}".format(logger.filter_secrets(config))) + +auth_type, username, password, server_url, is_ssl_check_disabled = get_credentials(config) +is_ssl_check_disabled = config.get("is_ssl_check_disabled", False) # Because no advanced parameter switch + +network_timer = PerformanceTimer() +processing_timer = PerformanceTimer() +processing_timer.start() + +output_dataset = dataiku.Dataset(output_names_stats[0]) +schema = [ + {'name': 'title', 'type': 'string'}, + {'name': 'template_name', 'type': 'string'}, + {'name': 'category_names', 'type': 'string'}, + {'name': 'path', 'type': 'string'}, + {'name': 'paths', 'type': 'string'}, + {'name': 'id', 'type': 'string'}, + {'name': 'url', 'type': 'string'}, + {'name': 'data_type', 'type': 'string'}, + {'name': 'summary_type', 'type': 'array'}, + {'name': 'boundary_type', 'type': 'string'}, + {'name': 'record_boundary_type', 'type': 'string'}, + {'name': 'summary_duration', 'type': 'string'}, + {'name': 'max_count', 'type': 'int'}, + +] +output_dataset.write_schema(schema) + +selectedAttributes = config.get("outputSelectedAttributes", []) +with output_dataset.get_writer() as writer: + for item in selectedAttributes: + if item.get("checked", True) is True: + writer.write_row_dict(item) + +processing_timer.stop() +logger.info("Overall timer:{}".format(processing_timer.get_report())) +logger.info("Network timer:{}".format(network_timer.get_report())) diff --git a/custom-recipes/pi-system-retrieve-list/recipe.json b/custom-recipes/pi-system-retrieve-list/recipe.json index a7425143..d30a1e49 100644 --- a/custom-recipes/pi-system-retrieve-list/recipe.json +++ b/custom-recipes/pi-system-retrieve-list/recipe.json @@ -105,7 +105,7 @@ "name": "use_batch_mode", "label": "Use batch mode", "type": "BOOLEAN", - "description": "", + "description": "Use to quickly retrieve small samples from multiple paths. ⚠️Not for large time ranges", "visibilityCondition": "model.show_advanced_parameters==true", "defaultValue": false }, diff --git a/custom-recipes/pi-system-retrieve-list/recipe.py b/custom-recipes/pi-system-retrieve-list/recipe.py index 88b4fbb8..b0f3c1cb 100644 --- a/custom-recipes/pi-system-retrieve-list/recipe.py +++ b/custom-recipes/pi-system-retrieve-list/recipe.py @@ -7,7 +7,8 @@ get_credentials, get_interpolated_parameters, normalize_af_path, get_combined_description, get_base_for_data_type, check_debug_mode, PerformanceTimer, get_max_count, check_must_convert_object_to_string, - convert_schema_objects_to_string, get_summary_parameters, get_advanced_parameters + convert_schema_objects_to_string, get_summary_parameters, get_advanced_parameters, + get_batch_parameters ) from osisoft_client import OSIsoftClient from osisoft_constants import OSIsoftConstants @@ -63,6 +64,8 @@ def get_step_value(item): record_boundary_type = config.get("record_boundary_type") if data_type == "RecordedData" else None summary_type, summary_duration = get_summary_parameters(config) do_duplicate_input_row = config.get("do_duplicate_input_row", False) +max_request_size, estimated_density, maximum_points_returned = get_batch_parameters(config) +max_time_to_retrieve_per_batch = estimated_density / maximum_points_returned #density per hour <- max time is in hour network_timer = PerformanceTimer() processing_timer = PerformanceTimer() @@ -150,7 +153,9 @@ def get_step_value(item): object_id=object_id, summary_type=summary_type, summary_duration=summary_duration, - endpoint_type="AF" + endpoint_type="AF", + estimated_density=estimated_density, + maximum_points_returned=maximum_points_returned ) batch_buffer_size = 0 buffer = [] diff --git a/js/pi-system_treecontroller.js b/js/pi-system_treecontroller.js new file mode 100644 index 00000000..6ae49016 --- /dev/null +++ b/js/pi-system_treecontroller.js @@ -0,0 +1,1203 @@ +const app = angular.module('piSystemTreeApp.module', []); + +const aggregateDataTypeFields = Object.freeze({ + data_type: { + label: 'Data type', + type: 'select', + defaultValue: 'RecordedData', + options: [ + { value: 'InterpolatedData', label: 'Interpolated' }, + { value: 'PlotData', label: 'Plot' }, + { value: 'RecordedData', label: 'Recorded' }, + { value: 'SummaryData', label: 'Summary' }, + { value: 'Value', label: 'Value' }, + { value: 'EndValue', label: 'End value' }, + ] + }, + aggregates: { + summary_type: { + label: 'Summary type', + type: 'multiselect', + dependsOn: ['data_type'], + defaultValue: [], + isVisible: function(attribute) { + return attribute.data_type === 'SummaryData'; + }, + options: [ + { value: 'Total', label: 'Total' }, + { value: 'Average', label: 'Average' }, + { value: 'Minimum', label: 'Minimum' }, + { value: 'Maximum', label: 'Maximum' }, + { value: 'Range', label: 'Range' }, + { value: 'StdDev', label: 'Standard deviation' }, + { value: 'PopulationStdDev', label: 'Population standard deviation' }, + { value: 'Count', label: 'Count' }, + { value: 'PercentGood', label: 'Percent good' }, + { value: 'TotalWithUOM', label: 'Total with UOM' }, + { value: 'All', label: 'All' }, + { value: 'AllForNonNumeric', label: 'All for non numeric' }, + ] + }, + boundary_type: { + label: 'Boundary type', + type: 'select', + dependsOn: ['data_type'], + defaultValue: 'Inside', + isVisible: function(attribute) { + return attribute.data_type === 'InterpolatedData'; + }, + options: [ + { value: 'Inside', label: 'Inside' }, + { value: 'Outside', label: 'Outside' }, + ] + }, + record_boundary_type: { + label: 'Boundary type', + type: 'select', + dependsOn: ['data_type'], + defaultValue: 'Inside', + isVisible: function(attribute) { + return attribute.data_type === 'RecordedData'; + }, + options: [ + { value: 'Inside', label: 'Inside' }, + { value: 'Interpolated', label: 'Interpolated' }, + { value: 'Outside', label: 'Outside' }, + ] + }, + summary_duration: { + label: 'Summary duration', + type: 'text', + dependsOn: ['data_type'], + defaultValue: '', + isVisible: function(attribute) { + return attribute.data_type === 'SummaryData'; + }, + }, + max_count: { + label: 'Max count', + type: 'number', + dependsOn: ['data_type'], + defaultValue: 10000, + isVisible: function(attribute) { + return ['PlotData', 'InterpolatedData', 'RecordedData'].includes(attribute.data_type); + }, + }, + } +}); + +//TODO: divide at least into a tree component + a results/right panel component + welcome component +const CheckboxStatus = Object.freeze({ + CHECKED: 'CHECKED', + UNCHECKED: 'UNCHECKED', + PARTIAL_CHECK: 'PARTIAL_CHECK', +}); + +app.service('TreeDataService', function() { + // This will store the shared tree data + this.treeData = []; + this.templateTreeData = []; + + // Optional: helper methods + this.setTreeData = function(data) { + this.treeData = data; + }; + + this.getTreeData = function() { + return this.treeData; + }; + + this.setTemplateTreeData = function(data) { + this.templateTreeData = data; + }; + + this.getTemplateTreeData = function() { + return this.templateTreeData; + }; +}); + +app.controller('AfExplorerFormCtrl', [ + '$scope', + '$stateParams', + '$q', + 'CodeMirrorSettingService', + 'TreeDataService', + function($scope, $stateParams, $q, CodeMirrorSettingService, TreeDataService) { + + $scope.paramDesc = { + 'parameterSetId': 'basic-auth', + 'mandatory': true + }; + + $scope.config.attributeList = $scope.config.attributeList || []; // la liste des attributs qui sont affichés sur le main panel à droite + $scope.config.outputSelectedAttributes = $scope.config.outputSelectedAttributes || []; // la liste des attributs qui sont séléctionnés pour être dans l'output dataset + $scope.config.searchMatchedElementPaths = $scope.config.searchMatchedElementPaths || []; // la liste pour highlighter les elements de la recherche + $scope.config.lastSearchedElementName = $scope.config.lastSearchedElementName || ""; + $scope.config.pendingTabContextReset = $scope.config.pendingTabContextReset || false; // indique le changement de tab template/element + $scope.config.selectedTemplateNames = $scope.config.selectedTemplateNames || []; // la liste des templates sélectionnés (checkbox cochée) parmi ceux affichés + + $scope.aggregateDataTypeFields = aggregateDataTypeFields; + $scope.attributeGroupSections = [ + { + key: 'attributesWithoutTemplate', + title: 'Elements', + emptyMessage: 'No attributes without template' + }, + { + key: 'attributesGroupedByTemplate', + title: 'Templates', + emptyMessage: 'No templated attributes' + } + ]; + + $scope.onAdvancedToggle = function() { + if (!$scope.config.show_advanced_parameters) { + $scope.config.is_ssl_check_disabled = false; + $scope.config.elements_max_count = null; + $scope.config.attributes_max_count = null; + } else { + if ($scope.config.elements_max_count === null || $scope.config.elements_max_count === undefined || $scope.config.elements_max_count === "") { + $scope.config.elements_max_count = 100; + } + if ($scope.config.attributes_max_count === null || $scope.config.attributes_max_count === undefined || $scope.config.attributes_max_count === "") { + $scope.config.attributes_max_count = 100; + } + } + }; + + $scope.init = function() { + $scope.config.show_advanced_parameters = $scope.config.show_advanced_parameters || false; + $scope.config.activeTab = $scope.config.activeTab || 'element'; + DataikuAPI.plugins.listAccessiblePresets('pi-system', $stateParams.projectKey, 'basic-auth').success(function(data) { + $scope.inlineParams = data.inlineParams; + $scope.inlinePluginParams = data.inlinePluginParams; + $scope.accessiblePresets = []; + if (data.definableInline) { + $scope.accessiblePresets.push({ + name: "INLINE", + label: "Manually defined", usable: true, + description: "Define values for these parameters" + }); + } + data.presets.forEach(function(p) { + $scope.accessiblePresets.push({ name: "PRESET " + p.name, label: p.name, usable: p.usable, description: p.description }); + }); + // TODO: why injection + $scope.accessibleParameterSetDescriptions = $scope.accessiblePresets.map(function(p) { + return p.description || 'No description'; + }); + }).error(setErrorInScope.bind($scope.errorScope)); + if ($scope.authConfigured() === true) { + const hasTreeData = Array.isArray($scope.config.treeData) && $scope.config.treeData.length > 0; + const hasTemplateTreeData = Array.isArray($scope.config.templateTreeData) && $scope.config.templateTreeData.length > 0; + $scope.authSectionVisible = !hasTreeData; + $scope.showTreeData = hasTreeData; + } + $scope.config.template = $scope.config.template || "-- Any --"; + $scope.onAdvancedToggle(); + }; + + $scope.getServers = function() { + $scope.callPythonDo({ parameterName: "server_name" }).then(function(data) { + $scope.server_name = data.choices; + }); + }; + $scope.getDatabases = function() { + $scope.callPythonDo({ parameterName: "database_name" }).then(function(data) { + $scope.database_name = data.choices; + }); + }; + + $scope.authSectionVisible = $scope.authSectionVisible || true; + + $scope.toggleAuthSection = function() { + $scope.authSectionVisible = !$scope.authSectionVisible; + }; + + $scope.authConfigured = function() { + return $scope.hasPreset() && !!$scope.config.database_name && !!$scope.config.server_name; + } + $scope.explore = function() { + const hasPreset = $scope.hasPreset(); + const hasServer = !!$scope.config.server_name; + const hasDatabase = !!$scope.config.database_name; + console.info("[LOGIN][UI] click", { + hasPreset: hasPreset, + hasServer: hasServer, + hasDatabase: hasDatabase + }); + + if (!$scope.authConfigured()) { + console.warn("[LOGIN][UI] blocked: missing required fields"); + return; + } + + console.info("[LOGIN][UI] dispatching login API calls", { + server_name: $scope.config.server_name, + database_name: $scope.config.database_name + }); + $scope.updateDatas().then( + function() { + $scope.showTreeData = true; + $scope.authSectionVisible = false; + console.info("[LOGIN][UI] success", { + tree_count: Array.isArray($scope.config.treeData) ? $scope.config.treeData.length : 0, + template_tree_count: Array.isArray($scope.config.templateTreeData) ? $scope.config.templateTreeData.length : 0 + }); + }, + function(error) { + $scope.showTreeData = false; + $scope.authSectionVisible = true; + console.error("[LOGIN][UI] failed", error); + } + ); + }; + + $scope.hasPreset = function() { + return $scope.config.credentials?.mode && $scope.config.credentials.mode !== 'NONE' && $scope.config.credentials.name + } + + $scope.cleanTree = function() { // utile quand on change de serveur ou de db dans la config + $scope.config.treeData = []; + $scope.config.clickedNodes = []; + $scope.config.attributeList = []; + $scope.config.outputSelectedAttributes = []; + $scope.config.searchMatchedElementPaths = []; + $scope.config.lastSearchedElementName = ""; + $scope.config.pendingTabContextReset = false; + $scope.config.selectedTemplateNames = []; + } + + $scope.resetDatasourceState = function() { // + $scope.server_name = []; + $scope.database_name = []; + $scope.config.server_name = null; + $scope.config.database_name = null; + $scope.config.templates = []; + $scope.config.templateTreeData = []; + $scope.config.attribute_categories = []; + $scope.config.element_categories = []; + $scope.config.loadedDatabaseName = null; + $scope.config.attributeList = []; + $scope.config.outputSelectedAttributes = []; + $scope.showTreeData = false; + $scope.cleanTree(); + }; + + $scope.onServerChanged = function() { + $scope.config.database_name = null; + $scope.config.templates = []; + $scope.config.templateTreeData = []; + $scope.config.attribute_categories = []; + $scope.config.element_categories = []; + $scope.config.loadedDatabaseName = null; + $scope.showTreeData = false; + $scope.cleanTree(); + $scope.getDatabases(); + }; + + $scope.onDatabaseChanged = function() { + $scope.config.templates = []; + $scope.config.templateTreeData = []; + $scope.config.attribute_categories = []; + $scope.config.element_categories = []; + $scope.config.loadedDatabaseName = null; + $scope.showTreeData = false; + $scope.cleanTree(); + }; + + let presetWatchInitialized = false; + // TODO: move this to an ng-change + $scope.$watchGroup( + [ + function() { + return $scope.config?.credentials?.mode ?? null; + }, + function() { + return $scope.config?.credentials?.name ?? null; + } + ], + function(newValues, oldValues) { + if (!presetWatchInitialized) { + presetWatchInitialized = true; + return; + } + + const mode = newValues[0]; + const name = newValues[1]; + const oldMode = oldValues ? oldValues[0] : null; + const oldName = oldValues ? oldValues[1] : null; + + if (mode === oldMode && name === oldName) { + return; + } + + $scope.resetDatasourceState(); + + if ($scope.hasPreset()) { + $scope.getServers(); + } + } + ); + + $scope.initializeTree = function() { + if (!$scope.config.treeData || $scope.config.treeData.length === 0) { + return $scope.callPythonDo({ method: "get_children_from_db", parent: $scope.config.database_name }).then(function(data) { + TreeDataService.setTreeData(data.choices); + $scope.config.treeData = TreeDataService.getTreeData(); + return data; + }); + } + return $q.when({ choices: $scope.config.treeData || [] }); + }; + + $scope.updateDatas = function() { + $scope.cleanTree(); + return $q.all([ + $scope.initializeTree(), + $scope.getTemplatesFromDB(), + $scope.getCategoriesFromDB() + ]).then(function(results) { + $scope.config.loadedDatabaseName = $scope.config.database_name || null; + return results; + }); + } + + $scope.getChildrenFromDB = function(item) { + console.log("ALX:gcfd:" + JSON.stringify(item)); + return $scope.callPythonDo({ method: "get_children_from_db", parent: item }) + .then(function(data) { + console.log("ALX:data1=" + JSON.stringify(data)); + item.children = data.choices; + item.children.forEach(child => { + child.expanded = false; + }); + markSearchResults(item.children, $scope.config.searchMatchedElementPaths || []); + console.log(item); + return item; + }); + } + + + $scope.getTemplatesFromDB = function() { + return $scope.callPythonDo({ method: "get_templates_from_db" }).then(function(data) { + $scope.config.templates = data.choices; + TreeDataService.setTemplateTreeData(data.choices); + $scope.config.templateTreeData = TreeDataService.getTemplateTreeData(); + }); + } + + function resetRightPanelForCurrentTabContext() { + $scope.config.attribute_name = ""; + $scope.config.clickedNodes = []; + $scope.config.attributeList = []; + $scope.config.searchMatchedElementPaths = []; + $scope.config.selectedTemplateNames = []; + if ($scope.config.activeTab === "element") { + $scope.config.template = "-- Any --"; + } else if ($scope.config.activeTab === "template") { + $scope.config.element_name = ""; + } + } + + function consumePendingTabContextReset() { // reset la main view après changement de tab + action sur le new tab + if (!$scope.config.pendingTabContextReset) { + return; + } + resetRightPanelForCurrentTabContext(); + $scope.config.pendingTabContextReset = false; + } + + $scope.setTab = function(tab) { + const previousTab = $scope.config.activeTab; + if (tab !== previousTab) { + $scope.config.pendingTabContextReset = true; + } + $scope.config.activeTab = tab; + }; + + $scope.getCategoriesFromDB = function() { + $scope.config.attribute_categories = []; + $scope.config.element_categories = []; + const attributeCategoriesPromise = $scope.callPythonDo({ method: "get_attribute_categories_from_db" }).then(function(data) { + $scope.config.attribute_categories = data.choices; + return data; + }); + const elementCategoriesPromise = $scope.callPythonDo({ method: "get_element_categories_from_db" }).then(function(data) { + $scope.config.element_categories = data.choices; + return data; + }); + return $q.all([attributeCategoriesPromise, elementCategoriesPromise]); + } + + $scope.doSearch = function(element_name, attribute_name) { + consumePendingTabContextReset(); + + const hasElementFilter = !!(element_name?.trim()); + const hadPreviousElementFilter = !!($scope.config.lastSearchedElementName?.trim()); + + // If user clears element filter after a scoped search, release previous click-based scope. + if (!hasElementFilter && hadPreviousElementFilter) { + $scope.config.clickedNodes = []; + } + + const hasClickedNodes = Array.isArray($scope.config.clickedNodes) && $scope.config.clickedNodes.length > 0; + const hasAttributeFilter = !!(attribute_name?.trim()); + const isRestrictedAttributeSearch = hasClickedNodes && hasAttributeFilter && !hasElementFilter; + const hasTemplateFilter = !!( + $scope.config.template && + $scope.config.template !== "-- Any --" + ); + const isTemplateScopedSearch = + hasTemplateFilter && + ($scope.config.activeTab === "template"); + const shouldDisplaySearchAttributesDirectly = + hasAttributeFilter || isTemplateScopedSearch; + $scope.config.lastSearchedElementName = element_name || ""; + if ($scope.config.activeTab === "template") { + $scope.config.selectedTemplateNames = getSelectedTemplateNamesFromClickedNodes(); + } else { + $scope.config.selectedTemplateNames = []; + } + const hasSelectedTemplateNodes = ( + $scope.config.activeTab === "template" && + Array.isArray($scope.config.selectedTemplateNames) && + $scope.config.selectedTemplateNames.length > 0 + ); + const shouldShowTemplateSelectionAttributes = hasSelectedTemplateNodes; + + if (!isRestrictedAttributeSearch) { + $scope.config.attributeList = []; + } + $scope.config.searchMatchedElementPaths = []; + // TODO: understand what this does + $scope.callPythonDo({ method: "do_search", element_name: element_name, attribute_name: attribute_name, root_tree: $scope.config.treeData }).then( + function(data) { + TreeDataService.setTreeData(data.choices); + $scope.config.treeData = TreeDataService.getTreeData(); + const matchedAttributes = data.attributes || []; + const matchedElementPaths = getMatchedElementPaths(matchedAttributes); + $scope.config.searchMatchedElementPaths = matchedElementPaths; + markSearchResults($scope.config.treeData, matchedElementPaths); + if ( + isRestrictedAttributeSearch || + shouldDisplaySearchAttributesDirectly || + shouldShowTemplateSelectionAttributes + ) { + applySearchAttributesToList(matchedAttributes); + } + } + ); + }; + + function applySearchAttributesToList(attributes) { + const seen = new Set(); + const deduped = []; + + attributes.forEach(attribute => { + if (!attribute?.path || seen.has(attribute.path)) { + return; + } + seen.add(attribute.path); + const attrCopy = { ...attribute }; + deduped.push(enrichAttribute(attrCopy)); + }); + + $scope.config.attributeList = deduped; + } + + function getMatchedElementPaths(attributes) { + const matchedPathSet = new Set(); + attributes.forEach(attribute => { + const fullPath = attribute?.path; + if (!fullPath || typeof fullPath !== "string") { + return; + } + const elementPath = fullPath.includes("|") ? fullPath.split("|")[0] : fullPath; + matchedPathSet.add(elementPath); + }); + return Array.from(matchedPathSet); + } + + + function collectTemplateTitlesByClickedUrls(nodes, clickedUrlSet, outputSet) { + if (!Array.isArray(nodes) || !clickedUrlSet || !outputSet) { + return; + } + + nodes.forEach(function(node) { + if (!node) { + return; + } + if ( + clickedUrlSet.has(node.url) && + node.type === "template" && + node.title && + node.title !== "-- Any --" + ) { + outputSet.add(node.title); + } + if (Array.isArray(node.children) && node.children.length > 0) { + collectTemplateTitlesByClickedUrls(node.children, clickedUrlSet, outputSet); + } + }); + } + + function getSelectedTemplateNamesFromClickedNodes() { + const clickedUrls = Array.isArray($scope.config.clickedNodes) + ? $scope.config.clickedNodes + : []; + if (!clickedUrls.length) { + return []; + } + + const selectedTemplateNames = new Set(); + collectTemplateTitlesByClickedUrls( + $scope.config.templateTreeData, + new Set(clickedUrls), + selectedTemplateNames + ); + return Array.from(selectedTemplateNames); + } + + function markSearchResults(nodes, matchedElementPaths) { + if (!Array.isArray(nodes)) { + return; + } + const matchedPathSet = new Set(matchedElementPaths || []); + + nodes.forEach(node => { + node.searchHighlighted = + node && + node.type !== "attribute" && + !!node.path && + matchedPathSet.has(node.path); + + if (Array.isArray(node.children) && node.children.length > 0) { + markSearchResults(node.children, matchedElementPaths); + } + }); + } + + $scope.onSearchInputKeydown = function($event) { + if ($event && ($event.key === "Enter" || $event.keyCode === 13)) { + $event.preventDefault(); + const targetId = $event.target?.id || ""; + if (targetId === "ReturnsName") { + $scope.searchFromElement(); + return; + } + $scope.doSearch($scope.config.element_name, $scope.config.attribute_name); + } + }; + + $scope.searchFromElement = function() { + if (!$scope.config) { + return; + } + + // Left search always resets right-side filter/template search. + $scope.config.clickedNodes = []; + $scope.config.selectedTemplateNames = []; + $scope.config.attribute_name = ""; + $scope.doSearch($scope.config.element_name, $scope.config.attribute_name); + }; + + function setAttributesChecked(attributes, isChecked) { + if (!Array.isArray(attributes)) { + return; + } + attributes.forEach(attribute => { + attribute.checked = !!isChecked; + if (isChecked) { + $scope.addAttributeToSelection(attribute); + } else { + $scope.removeAttributeFromSelection(attribute); + } + }); + } + + $scope.toggleSelectAllGroupedAttributes = function(groupedAttributes) { + const shouldRemove = groupedAttributes.checked === CheckboxStatus.CHECKED; + groupedAttributes.groups.forEach((group) => { + group.attributes.forEach((aggregatedAttribute) => { + aggregatedAttribute.attributes.forEach((underlyingAttribute) => { + if (shouldRemove) { + $scope.removeAttributeFromSelection(underlyingAttribute); + return; + } + $scope.addAttributeToSelection(underlyingAttribute); + }); + }); + }); + }; + + $scope.checkAttribute = function(attributeList) { + const shouldRemove = attributeList.checked === CheckboxStatus.CHECKED; + attributeList.attributes.forEach((attribute) => { + if (shouldRemove) { + $scope.removeAttributeFromSelection(attribute); + return; + } + $scope.addAttributeToSelection(attribute); + } + ) + }; + + $scope.checkTemplate = function(template) { + const shouldRemove = template.checked === CheckboxStatus.CHECKED; + template.attributes.forEach((aggregatedAttribute) => { + aggregatedAttribute.attributes.forEach((underlyingAttribute) => { + if (shouldRemove) { + $scope.removeAttributeFromSelection(underlyingAttribute); + return; + } + $scope.addAttributeToSelection(underlyingAttribute); + }); + } + ) + }; + + // TODO: mark as loaded elements and replace this logic + function hasAttributeChildren(node) { + if (!Array.isArray(node.children) || node.children.length === 0) { + return false + } + return node.children.some(child => child.type === "attribute"); + } + + function getChildren(node) { + if (hasAttributeChildren(node)) { + return Promise.resolve(node); + } + return $scope.getChildrenFromDB(node); + } + + function stopDisplayingAttributes(node) { + // It is for now possible to stop displaying an element that was not loaded because of weak links + // patching it by loading the element before stopping to display it + // TODO: replace by weak link single loading logic + getChildren(node).then( node => { + const nodeAttributeChildrenPaths = node.children.filter(child => child.type === "attribute" && child.path) + .map(child => child.path) + if (!nodeAttributeChildrenPaths.length) { + return; + } + + $scope.config.attributeList = $scope.config.attributeList.filter( + attr => !nodeAttributeChildrenPaths.includes(attr.path) + ); + }); + } + + $scope.toggleDisplayAttributes = function(node, add = true) { + if (!add) { + stopDisplayingAttributes(node); + return; + } + // TODO: refacto + if (node.type === "element" && !hasAttributeChildren(node)) { + $scope.config.template = "-- Any --"; + $scope.getChildrenFromDB(node).then(newNode => { + processNode(newNode); + }); + } else if (node.type === "template") { + const selectedTemplateNames = getSelectedTemplateNamesFromClickedNodes(); + if (!selectedTemplateNames.length) { + $scope.config.template = "-- Any --"; + $scope.config.attributeList = []; + $scope.config.searchMatchedElementPaths = []; + return; + } + + // Keep previous single-template behavior in config when only one is selected. + // For multi-select, backend will use selectedTemplateNames. + $scope.config.template = selectedTemplateNames.length === 1 + ? selectedTemplateNames[0] + : "-- Any --"; + $scope.config.element_name = "*"; + $scope.doSearch($scope.config.element_name, $scope.config.attribute_name); + } else { + processNode(node); + } + } + + // Merge frontend data and saved output with loaded attributes + function enrichAttribute(attribute, parentNode) { + // TODO: check this makes sense, since selectedOutput is persisted and so newly loaded attributes should not be found in it + const selectedAttribute = $scope.config.outputSelectedAttributes.find(attr => attr.path === attribute.path); + attribute.checked = !!(selectedAttribute); + attribute.parent_element = parentNode.title; + attribute.data_type = selectedAttribute?.data_type ? selectedAttribute.data_type : $scope.aggregateDataTypeFields.data_type.defaultValue; + Object.entries($scope.aggregateDataTypeFields.aggregates).forEach(([aggregateName, aggregate]) => { + if ((selectedAttribute?.[aggregateName] === undefined || selectedAttribute?.[aggregateName] === null) && aggregate.isVisible(attribute)) { + attribute[aggregateName] = aggregate.defaultValue; + } else if (selectedAttribute?.[aggregateName] !== null && selectedAttribute?.[aggregateName] !== undefined) { + attribute[aggregateName] = selectedAttribute?.[aggregateName]; + } else { + attribute[aggregateName] = null; + } + }); + return attribute; + } + + function processNode(node) { + const hasAttributeFilter = !!($scope.config.attribute_name?.trim()); + const parentTemplateName = node?.template_name ? node.template_name : null; + + node.children.forEach(child => { + if (child.type === "attribute") { + if (!child.parent_template_name && parentTemplateName) { + child.parent_template_name = parentTemplateName; + } + if (hasAttributeFilter && !attributeMatchesCurrentSearch(child)) { + return; + } + const isAlreadyPresent = $scope.config.attributeList.some(attr => attr.path === child.path); + if (!isAlreadyPresent) { + $scope.config.attributeList.push(enrichAttribute(child, node)); + } + } + }); + } + + function getAggregateNames() { + return Object.keys($scope.aggregateDataTypeFields.aggregates); + } + + function getAggregateValuesKey(aggregateName) { + return aggregateName + 's'; + } + + function stringArraysEqual(a, b) { + if (!a || !b) { + return false; + } + return a.length === b.length && + [...a].sort().every((v, i) => v === [...b].sort()[i]); + } + + + // reset all aggregates on change data type + function resetAggregate(attribute) { + Object.entries($scope.aggregateDataTypeFields.aggregates).forEach(([aggregateName, aggregate]) => { + if (!aggregate.isVisible(attribute)) { + attribute[aggregateName] = null + return; + } + attribute[aggregateName] = aggregate.defaultValue; + } + ) + } + + $scope.updateMergedAttributeDataType = function(mergedAttribute) { + mergedAttribute.attributes.forEach(attribute => { + attribute.data_type = mergedAttribute.data_type; + resetAggregate(attribute); + if (attribute.checked) { + $scope.updateAttributeInSelection(attribute) + } + }); + } + + $scope.updateMergedAttributeAggregate = function(mergedAttribute) { + const aggregateNames = getAggregateNames(); + + mergedAttribute.attributes.forEach(attribute => { + aggregateNames.forEach(aggregateName => { + // TODO: check not necessary to copy to avoid arrays being linked + attribute[aggregateName] = mergedAttribute[aggregateName]; + }); + if (attribute.checked) { + $scope.updateAttributeInSelection(attribute) + } + }); + }; + + function groupDuplicatedAttributesAcrossGroup(groupKey) { + return (acc, attr) => { + // TODO: switch to id + const key = attr[groupKey] + "::" + attr.title; + + if (!acc[key]) { + acc[key] = { + title: attr.title, + description: attr.description, + group: attr[groupKey], + template_names: [], + parent_elements: [], + checked: null, // Used to determine UI checkbox state + allChecked: attr.checked, + attributes: [], + checkStates: [], + paths: [], + data_type: attr.data_type, + data_types: [], + }; + + getAggregateNames().forEach(aggregateName => { + acc[key][aggregateName] = attr[aggregateName]; + acc[key][getAggregateValuesKey(aggregateName)] = []; + }); + } + + acc[key].checkStates.push(attr.checked) + acc[key].template_names.push(attr.template_name) + acc[key].paths.push(attr.path) + acc[key].parent_elements.push(attr.parent_element); + acc[key].checked = getCheckboxStatus(acc[key].checkStates); // TODO maybe move out + acc[key].allChecked = acc[key].allChecked && attr.checked + acc[key].attributes.push(attr); + acc[key].data_types.push(attr.data_type); + + if (acc[key].data_type !== attr.data_type) { + acc[key].data_type = null; + } + + getAggregateNames().forEach(aggregateName => { + acc[key][getAggregateValuesKey(aggregateName)].push(attr[aggregateName]); + if ($scope.aggregateDataTypeFields.aggregates[aggregateName].type === 'multiselect') { + if (!stringArraysEqual(acc[key][aggregateName], attr[aggregateName])) { + acc[key][aggregateName] = []; + } + return; + } + if (acc[key][aggregateName] !== attr[aggregateName]) { + acc[key][aggregateName] = null; + } + }); + + return acc + } + } + + function groupAttributes() { + return (acc, attr) => { + const key = attr.group; + if (!acc[key]) { + acc[key] = { + group_name: attr.group, + allChecked: attr.checked, + checked: CheckboxStatus.UNCHECKED, // Used to determine UI checkbox state + attributes: [], + checkStates: [] + } + } + + acc[key].checkStates.push(...attr.checkStates) + acc[key].checked = getCheckboxStatus(acc[key].checkStates); + acc[key].allChecked = acc[key].allChecked && attr.allChecked; + acc[key].attributes.push(attr); + return acc; + } + } + + function buildAggregatedAttributes(attributes, groupKey) { + const deduplicatedAttributes = Object.values(attributes.reduce(groupDuplicatedAttributesAcrossGroup(groupKey), {})); + return Object.values(deduplicatedAttributes.reduce(groupAttributes(), {})); + } + + function splitAttributesByTemplatePresence(attributes) { + return attributes.reduce( + (accumulator, attribute) => { + // TODO: make the attribute have a template name even if no template + const bucket = attribute?.template_name + ? 'attributesWithTemplate' + : 'attributesWithoutTemplate'; + accumulator[bucket].push(attribute); + return accumulator; + }, + { attributesWithoutTemplate: [], attributesWithTemplate: [] } + ); + } + + function buildGroupedAttributesResult(attributes, groupKey) { + const groups = buildAggregatedAttributes(attributes, groupKey); + return { + allChecked: groups.length > 0 && groups.every(group => group.allChecked), + checked: getCheckboxStatus(groups.reduce((acc, group) => acc.concat(group.checkStates), [])), + groups: groups + } + } + + $scope.buildGroupedAttributes = function() { + const splitAttributes = splitAttributesByTemplatePresence($scope.config.attributeList); + return { + attributesWithoutTemplate: buildGroupedAttributesResult( + splitAttributes.attributesWithoutTemplate, + 'parent_element' + ), + attributesGroupedByTemplate: buildGroupedAttributesResult( + splitAttributes.attributesWithTemplate, + 'template_name' + ) + }; + } + + function getCheckboxStatus(checkboxStatuses) { + if (!checkboxStatuses.length) { + return CheckboxStatus.UNCHECKED; + } + if (checkboxStatuses.every(Boolean)) { + return CheckboxStatus.CHECKED; + } else if (checkboxStatuses.some(Boolean)) { + return CheckboxStatus.PARTIAL_CHECK; + } + return CheckboxStatus.UNCHECKED; + } + + // TODO: try to move it to a callback of some kind (will work with a component) + $scope.$watch('config.attributeList', function(newVal, oldVal) { + $scope.groupedAttributes = $scope.buildGroupedAttributes(); + }, true); + + + function attributeMatchesCurrentSearch(attribute) { + const rawFilter = ($scope.config.attribute_name || "").trim(); + if (!rawFilter) { + return true; + } + + const attributeTitle = (attribute?.title ? attribute.title : "").toLowerCase(); + const filter = rawFilter.toLowerCase(); + + if (filter.includes("*")) { + const regexPattern = "^" + escapeRegex(filter).replace(/\\\*/g, ".*") + "$"; + return new RegExp(regexPattern).test(attributeTitle); + } + + return attributeTitle.includes(filter); + } + + function escapeRegex(input) { + return input.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); + } + + $scope.addAttributeToSelection = function(attribute) { + const index = $scope.config.outputSelectedAttributes.findIndex(attr => attr.path === attribute.path); + if (index !== -1) { + console.warn("Cannot add attribute to selection because already present", attribute); + return; + } + attribute.checked = true; + $scope.config.outputSelectedAttributes.push(attribute); + console.log("Removed attribute from selection", attribute); + } + + $scope.removeAttributeFromSelection = function(attribute) { + const index = $scope.config.outputSelectedAttributes.findIndex(attr => attr.path === attribute.path); + if (index === -1) { + console.warn("Cannot remove attribute from selection because not present", attribute); + return; + } + attribute.checked = false; + $scope.config.outputSelectedAttributes.splice(index, 1); + console.log("Removed attribute from selection", attribute); + } + + $scope.updateAttributeInSelection = function(attribute) { + const index = $scope.config.outputSelectedAttributes.findIndex(attr => attr.path === attribute.path); + if (index === -1) { + console.warn("Cannot update attribute in selection because not present", attribute); + return; + } + $scope.config.outputSelectedAttributes[index] = attribute; + } + + }]); + + +app.component('treeNode', { + bindings: { + node: '=', + getChildrenFromDb: '<', + toggleDisplayAttributes: '<', + config: '<', + }, + + controllerAs: 'ctrl', + + controller: function() { + const ctrl = this; + + function consumePendingTabContextReset() { + if (!ctrl.config?.pendingTabContextReset) { + return; + } + + ctrl.config.attribute_name = ""; + ctrl.config.clickedNodes = []; + ctrl.config.attributeList = []; + ctrl.config.searchMatchedElementPaths = []; + + if (ctrl.config.activeTab === "element") { + ctrl.config.template = "-- Any --"; + } else if (ctrl.config.activeTab === "template") { + ctrl.config.element_name = ""; + } + + ctrl.config.pendingTabContextReset = false; + } + + function findNodeByUrl(nodes, targetUrl) { + if (!Array.isArray(nodes) || !targetUrl) { + return null; + } + + for (let i = 0; i < nodes.length; i += 1) { + const node = nodes[i]; + if (!node) { + continue; + } + if (node.url === targetUrl) { + return node; + } + const childMatch = findNodeByUrl(node.children, targetUrl); + if (childMatch) { + return childMatch; + } + } + + return null; + } + + // TODO: understand why the logic is different from toggleDisplayAttributes (merge them if possible) + function rebuildAttributesFromClickedNodes() { + const clickedUrls = Array.isArray(ctrl.config?.clickedNodes) + ? ctrl.config.clickedNodes + : []; + + ctrl.config.attributeList = []; + + if (!clickedUrls.length) { + return; + } + + clickedUrls.forEach(function(url) { + const node = + findNodeByUrl(ctrl.config.treeData, url) || + findNodeByUrl(ctrl.config.templateTreeData, url); + if (node) { + ctrl.toggleDisplayAttributes(node); + } + }); + } + + ctrl.hasRenderableChildren = function(node) { + if (!node || !Array.isArray(node.children) || !node.children.length) { + return false; + } + return node.children.some(function(child) { + return child && child.type !== 'attribute'; + }); + }; + + ctrl.toggleExpand = function(node, $event) { + if ($event) { + $event.stopPropagation(); + } + // Loading children before toggling the node + if (!node.expanded && (!node.children?.length || !ctrl.hasRenderableChildren(node))) { + ctrl.getChildrenFromDb(node).then(() => { + node.expanded = true; + }); + return; + } + node.expanded = !node.expanded; + }; + + ctrl.onNodeClick = function(node) { + consumePendingTabContextReset(); + + // TODO: factorize this check + const hasActiveAttributeSearch = !!( + ctrl.config?.attribute_name?.trim() + ); + + // Keep right-side attribute search when active so multi-node clicks can + // enrich results with the same filter (ex: "Load" on California + Fresno). + // TODO: understand why we need a reset if the attribute search is empty + if (!hasActiveAttributeSearch) { + ctrl.config.attribute_name = ""; + } + if (node?.type === "element") { + // TODO: factorize this reset + ctrl.config.template = "-- Any --"; + } + + const indexClickedNode = ctrl.config.clickedNodes.indexOf(node.url); + const nodeAlreadySelected = indexClickedNode > -1; + // If the node is already clicked, remove it from clicked nodes - else add it + if (nodeAlreadySelected) { + ctrl.config.clickedNodes.splice(indexClickedNode, 1); + } else { + ctrl.config.clickedNodes.push(node.url); + } + + // TODO: split element/template logic + if (node?.type === "template") { + // Template clicks should always rebuild right-side content from the full template selection. + ctrl.toggleDisplayAttributes(node); + console.log("ctrl.config.clickedNodes: " + JSON.stringify(ctrl.config.clickedNodes)); + return; + } + + // TODO: understand why this is mutually exclusive + if (hasActiveAttributeSearch) { + rebuildAttributesFromClickedNodes(); + } else { + ctrl.toggleDisplayAttributes(node, !nodeAlreadySelected); + } + + console.log("ctrl.config.clickedNodes: " + JSON.stringify(ctrl.config.clickedNodes)); + }; + + ctrl.isNodeClicked = function(node) { + // the click is entirely based on node.url + return ctrl.config.clickedNodes.includes(node.url); + }; + + ctrl.isSearchResult = function(node) { + return !!node.searchHighlighted; + }; + }, + templateUrl: "/plugins/pi-system/resource/tree-node.html" +}); + +// TODO: see if cleaner architecture +app.directive('attributeTableRow', function() { + return { + restrict: 'A', + scope: { + mergedAttribute: '=', + aggregateDataTypeFields: '<', + onCheckAttribute: '&', + onUpdateDataType: '&', + onUpdateAggregate: '&', + }, + bindToController: true, + controllerAs: 'ctrl', + controller: function() { + }, + templateUrl: "/plugins/pi-system/resource/attribute-table-row.html" + }; +}); + +app.directive('indeterminate', function() { + return { + restrict: 'A', + link: function(scope, element, attrs) { + if (attrs.indeterminate === CheckboxStatus.PARTIAL_CHECK) { + element[0].indeterminate = true; + } + + scope.$watch(attrs.indeterminate, function(checkStatus) { + if (checkStatus === CheckboxStatus.PARTIAL_CHECK) { + element[0].indeterminate = true; + return; + } + element[0].indeterminate = false; + }, true); + } + }; +}); diff --git a/parameter-sets/basic-auth/parameter-set.json b/parameter-sets/basic-auth/parameter-set.json index 13e76de8..70c278c9 100644 --- a/parameter-sets/basic-auth/parameter-set.json +++ b/parameter-sets/basic-auth/parameter-set.json @@ -50,6 +50,28 @@ "description": "(optional)", "defaultValue": "" }, + { + "name": "max_request_size", + "label": "Maximum request size", + "type": "INT", + "description": "", + "defaultValue": 1000 + }, + { + "name": "estimated_density", + "label": "Estimated point density", + "type": "DOUBLE", + "description": "points/hour", + "defaultValue": 2 + }, + { + "name": "maximum_points_returned", + "label": "Maximum points return", + "type": "INT", + "description": "Target optimum number of points returned by batch. Calculated based on point density.", + "defaultValue": 1000000, + "minI": 1 + }, { "name": "osisoft_basic", "type": "CREDENTIAL_REQUEST", diff --git a/plugin.json b/plugin.json index 8daebaf9..4494e851 100644 --- a/plugin.json +++ b/plugin.json @@ -1,6 +1,6 @@ { "id": "pi-system", - "version": "1.4.0", + "version": "1.4.2", "meta": { "label": "PI System", "description": "Retrieve data from your OSIsoft PI System servers", diff --git a/python-connectors/pi-system_hierarchy/connector.json b/python-connectors/pi-system_hierarchy/connector.json new file mode 100644 index 00000000..f66a4ed6 --- /dev/null +++ b/python-connectors/pi-system_hierarchy/connector.json @@ -0,0 +1,91 @@ +{ + "meta" : { + "label": "AF Hierarchy", + "description": "", + "icon": "icon-pi-system icon-cogs" + }, + "readable": true, + "writable": false, + "supportAppend": false, + "kind": "PYTHON", + "paramsPythonSetup": "browse_event_frames.py", + "params": [ + { + "name": "credentials", + "label": "User preset", + "type": "PRESET", + "parameterSetId": "basic-auth" + }, + { + "name": "show_advanced_parameters", + "label": " ", + "type": "BOOLEAN", + "description": "Show advanced parameters", + "defaultValue": true + }, + { + "name": "server_url", + "label": "Server URL", + "visibilityCondition": "model.show_advanced_parameters==true", + "type": "STRING", + "description": "https://my_server:8082", + "defaultValue": "" + }, + { + "name": "is_ssl_check_disabled", + "label": " ", + "visibilityCondition": "model.show_advanced_parameters==true", + "type": "BOOLEAN", + "description": "Disable SSL check", + "defaultValue": false + }, + { + "name": "ssl_cert_path", + "label": "Path to SSL certificate", + "type": "STRING", + "description": "(optional)", + "visibilityCondition": "model.show_advanced_parameters==true && model.is_ssl_check_disabled==false", + "mandatory": false + }, + { + "name": "is_debug_mode", + "label": " ", + "visibilityCondition": "model.show_advanced_parameters==true", + "type": "BOOLEAN", + "description": "Verbose logging", + "defaultValue": false + }, + { + "name": "use_batch_mode", + "label": " ", + "type": "BOOLEAN", + "description": "Use batch mode", + "visibilityCondition": "model.show_advanced_parameters==true", + "defaultValue": true + }, + { + "name": "batch_size", + "label": " ", + "type": "INT", + "description": "Batch size", + "visibilityCondition": "model.show_advanced_parameters==true && model.use_batch_mode==true", + "minI": 1, + "defaultValue": 500 + }, + { + "name": "server_name", + "label": "Server name", + "type": "SELECT", + "description": "", + "getChoicesFromPython": true + }, + { + "name": "database_name", + "label": "Database name", + "type": "SELECT", + "description": "", + "visibilityCondition": "model.server_name.length>=0", + "getChoicesFromPython": true + } + ] +} diff --git a/python-connectors/pi-system_hierarchy/connector.py b/python-connectors/pi-system_hierarchy/connector.py new file mode 100644 index 00000000..fac8d137 --- /dev/null +++ b/python-connectors/pi-system_hierarchy/connector.py @@ -0,0 +1,259 @@ +import datetime +from dataiku.connector import Connector +from osisoft_client import OSIsoftClient +from safe_logger import SafeLogger +from osisoft_plugin_common import ( + RecordsLimit, get_credentials, + check_debug_mode, PerformanceTimer +) +from osisoft_constants import OSIsoftConstants + + +logger = SafeLogger("PI System plugin", ["user", "password"]) + + +class HierarchyConnector(Connector): + + def __init__(self, config, plugin_config): + Connector.__init__(self, config, plugin_config) + + logger.info("Attribute search v{} initialization with config={}, plugin_config={}".format( + OSIsoftConstants.PLUGIN_VERSION, logger.filter_secrets(config), logger.filter_secrets(plugin_config) + )) + + auth_type, username, password, server_url, is_ssl_check_disabled = get_credentials(config) + is_debug_mode = check_debug_mode(config) + self.database_endpoint = config.get("database_name") + + self.network_timer = PerformanceTimer() + self.client = OSIsoftClient( + server_url, auth_type, username, password, + is_ssl_check_disabled=is_ssl_check_disabled, + is_debug_mode=is_debug_mode, + network_timer=self.network_timer + ) + self.use_batch_mode = config.get("use_batch_mode", False) + self.batch_size = config.get("batch_size", 500) + + def get_read_schema(self): + return None + + def generate_rows(self, dataset_schema=None, dataset_partitioning=None, + partition_id=None, records_limit = -1): + limit = RecordsLimit(records_limit) + start_time = datetime.datetime.now() + + headers = self.client.get_requests_headers() + json_response = self.client.get(url=self.database_endpoint, headers=headers, params={}, error_source="traverse") + server_name = json_response.get("ExtendedProperties", {}).get("DefaultPIServer", {}).get("Value", "Unknown server name") + + if self.use_batch_mode: + for item in self.batch_next_item(json_response, parent=server_name, type="Database"): + if limit.is_reached(): + break + yield item + else: + next_url = self.client.extract_link_with_key(json_response, "Elements") + for item in self.recurse_next_item(next_url): + if limit.is_reached(): + break + yield item + end_time = datetime.datetime.now() + duration = end_time - start_time + logger.info("generate_rows overall duration = {}s".format(duration.microseconds/1000000 + duration.seconds)) + logger.info("Network timer:{}".format(self.network_timer.get_report())) + + def recurse_next_item(self, next_url, parent=None, type=None): + logger.info("recurse_next_item") + type = type or "Elements" + headers = self.client.get_requests_headers() + json_response = self.client.get(url=next_url, headers=headers, params={}, error_source="recurse") + items = json_response.get("Items") + if not items: + return + for item in items: + parent_path = item.get("Path") + link_to_attributes = self.client.extract_link_with_key(item, "Attributes") + if link_to_attributes: + for attribute in self.recurse_next_item(link_to_attributes, parent=parent_path, type="Attribute"): + yield attribute + link_to_elements = self.client.extract_link_with_key(item, "Elements") + if link_to_elements: + for element in self.recurse_next_item(link_to_elements, parent=parent_path, type="Element"): + yield element + yield { + "ItemType": type, + "Name": item.get("Name"), + "Type": item.get("Type"), + "Description": item.get("Description"), + "Path": item.get("Path"), + "Parent": parent, + "DefaultUnitsName": item.get("DefaultUnitsName"), + "TemplateName": item.get("TemplateName"), + "CategoryNames": item.get("CategoryNames"), + "ExtendedProperties": item.get("ExtendedProperties"), + "Step": item.get("Step"), + "WebId": item.get("WebId"), + "Id": item.get("Id") + } + + def batch_next_item(self, next_item, parent=None, type=None): + todo_list = [] + todo_list.append( + { + "url": self.client.extract_link_with_key(next_item, "Elements"), + "parent": "\\\\" + parent + "\\" + next_item.get("Name"), + "type": "Database" + } + ) + batch_requests_parameters= [] + parent_of_batched_items = [] + while todo_list: + item = todo_list.pop() + request_kwargs = { + "url": item.get("url"), + "headers": self.client.get_requests_headers() + } + batch_requests_parameters.append(request_kwargs) + parent_of_batched_items.append(item.get("parent")) + if not todo_list or len(batch_requests_parameters) > self.batch_size: + json_responses = self.client._batch_requests(batch_requests_parameters) + batch_requests_parameters = [] + for parent_of_batched_item, json_response in zip(parent_of_batched_items, json_responses): + response_content = json_response.get("Content", {}) + links = response_content.get("Links", {}) + next_link = links.get("Next", {}) + # do something if there is a next link... + if next_link: + todo_list.append( + { + "url": next_link + } + ) + retrieved_items = response_content.get(OSIsoftConstants.API_ITEM_KEY, []) + for retrieved_item in retrieved_items: + retrieved_item_path = retrieved_item.get("Path") + elements_url = self.client.extract_link_with_key(retrieved_item, "Elements") + attributes_url = self.client.extract_link_with_key(retrieved_item, "Attributes") + if elements_url: + todo_list.append( + { + "url": elements_url, + "type": "Element", + "parent": parent_of_batched_item + "\\" + retrieved_item.get("Name") + } + ) + if attributes_url: + todo_list.append( + { + "url": attributes_url, + "type": "Attribute", + "parent": parent_of_batched_item + "\\" + retrieved_item.get("Name") + } + ) + yield { + "ItemType": type, + "Name": retrieved_item.get("Name"), + "Type": retrieved_item.get("Type"), + "Description": retrieved_item.get("Description"), + "Path": retrieved_item.get("Path"), + "LinkPath": "{}\\{}".format(parent_of_batched_item, retrieved_item.get("Name")), + "Parent": parent_of_batched_item, + "DefaultUnitsName": retrieved_item.get("DefaultUnitsName"), + "TemplateName": retrieved_item.get("TemplateName"), + "CategoryNames": retrieved_item.get("CategoryNames"), + "ExtendedProperties": retrieved_item.get("ExtendedProperties"), + "Step": retrieved_item.get("Step"), + "WebId": retrieved_item.get("WebId"), + "Id": retrieved_item.get("Id") + } + parent_of_batched_items = [] + + + def batch_recurse_next_item(self, next_items, parents=None, type=None): + # logger.info("batch_recurse_next_item") + if not isinstance(next_items, list): + next_items = [next_items] + if not isinstance(parents, list): + parents = [parents] + batch_requests_parameters= [] + types = [] + items_parents_names = [] + for next_item in next_items: + next_item_name = next_item.get("Path") + next_elements_url = self.client.extract_link_with_key(next_item, "Elements") + if next_elements_url: + request_kwargs = { + "url": next_elements_url, + "headers": self.client.get_requests_headers() + } + batch_requests_parameters.append(request_kwargs) + types.append("Element") + items_parents_names.append(next_item_name) + next_attributes_url = self.client.extract_link_with_key(next_item, "Attributes") + if next_attributes_url: + request_kwargs = { + "url": next_attributes_url, + "headers": self.client.get_requests_headers() + } + batch_requests_parameters.append(request_kwargs) + types.append("Attribute") + items_parents_names.append(next_item_name) + if batch_requests_parameters: + json_responses = self.client._batch_requests(batch_requests_parameters) + # for json_response in json_responses: + # # Here we process recurse based on each response in the batch + # # Instead we could process all responses and batch all of them in one go... + # response_content = json_response.get("Content", {}) + # if OSIsoftConstants.DKU_ERROR_KEY in response_content: + # # Do something ? + # pass + # items = response_content.get(OSIsoftConstants.API_ITEM_KEY, []) + # batched_items = self.batch_recurse_next_item(items) + # for item in batched_items: + # yield item + # approach 2: + next_batch_items = [] + for json_response in json_responses: + response_content = json_response.get("Content", {}) + links = response_content.get("Links", {}) + next_link = links.get("Next", {}) + # do something if there is a next link... + items = response_content.get(OSIsoftConstants.API_ITEM_KEY, []) + next_batch_items.extend(items) + batched_items = self.batch_recurse_next_item(next_batch_items, parents=items_parents_names) + for item in batched_items: + yield item + + for item, parent in zip(next_items, parents): + yield { + "ItemType": type, + "Name": item.get("Name"), + "Type": item.get("Type"), + "Description": item.get("Description"), + "Path": item.get("Path"), + "Parent": parent, + "DefaultUnitsName": item.get("DefaultUnitsName"), + "TemplateName": item.get("TemplateName"), + "CategoryNames": item.get("CategoryNames"), + "ExtendedProperties": item.get("ExtendedProperties"), + "Step": item.get("Step"), + "WebId": item.get("WebId"), + "Id": item.get("Id") + } + + def get_writer(self, dataset_schema=None, dataset_partitioning=None, + partition_id=None, write_mode="OVERWRITE"): + raise NotImplementedError + + def get_partitioning(self): + raise NotImplementedError + + def list_partitions(self, partitioning): + return [] + + def partition_exists(self, partitioning, partition_id): + raise NotImplementedError + + def get_records_count(self, partitioning=None, partition_id=None): + raise NotImplementedError diff --git a/python-lib/osisoft_client.py b/python-lib/osisoft_client.py index d6d4d106..1f06024a 100644 --- a/python-lib/osisoft_client.py +++ b/python-lib/osisoft_client.py @@ -3,6 +3,7 @@ import copy import json import simplejson +import re from datetime import datetime from requests_ntlm import HttpNtlmAuth from osisoft_constants import OSIsoftConstants @@ -10,8 +11,10 @@ from osisoft_plugin_common import ( assert_server_url_ok, build_requests_params, is_filtered_out, is_server_throttling, escape, epoch_to_iso, - iso_to_epoch, RecordsLimit, is_iso8601, get_next_page_url, change_key_in_dict + iso_to_epoch, RecordsLimit, is_iso8601, get_next_page_url, change_key_in_dict, + BatchTimeCounter ) +from osisoft_plugin_common import get_item_details from osisoft_pagination import OffsetPagination from safe_logger import SafeLogger @@ -28,6 +31,7 @@ class OSIsoftClient(object): def __init__(self, server_url, auth_type, username, password, is_ssl_check_disabled=False, can_raise=True, is_debug_mode=False, network_timer=None): if can_raise: assert_server_url_ok(server_url) + requests.packages.urllib3.disable_warnings() self.session = requests.Session() self.session.auth = self.get_auth(auth_type, username, password) self.session.verify = (not is_ssl_check_disabled) @@ -243,7 +247,10 @@ def get_rows_from_webid(self, webid, data_type, **kwargs): def get_rows_from_webids(self, input_rows, data_type, **kwargs): endpoint_type = kwargs.get("endpoint_type", "event_frames") batch_size = kwargs.get("batch_size", 500) - + estimated_density = kwargs.get("estimated_density", 500) + maximum_points_returned = kwargs.get("maximum_points_returned", 500) + max_time_to_retrieve_per_batch = maximum_points_returned / estimated_density + batch_time = BatchTimeCounter(max_time_to_retrieve_per_batch) batch_requests_parameters = [] number_processed_webids = 0 number_of_webids_to_process = len(input_rows) @@ -259,14 +266,18 @@ def get_rows_from_webids(self, input_rows, data_type, **kwargs): else: webid = input_row url = self.endpoint.get_data_from_webid_url(endpoint_type, data_type, webid) + start_date = kwargs.get("start_date") + end_date = kwargs.get("end_date") + interval = kwargs.get("interval") requests_kwargs = self.generic_get_kwargs(**kwargs) + batch_time.add(start_date, end_date, interval) requests_kwargs['url'] = build_query_string(url, requests_kwargs.get("params")) web_ids.append(webid) event_start_times.append(event_start_time) event_end_times.append(event_end_time) batch_requests_parameters.append(requests_kwargs) number_processed_webids += 1 - if (len(batch_requests_parameters) >= batch_size) or (number_processed_webids == number_of_webids_to_process): + if (len(batch_requests_parameters) >= batch_size) or (number_processed_webids == number_of_webids_to_process) or batch_time.is_batch_full(): json_responses = self._batch_requests(batch_requests_parameters) batch_requests_parameters = [] response_index = 0 @@ -296,7 +307,12 @@ def _batch_requests(self, batch_requests_parameters, method=None): batch_endpoint = self.endpoint.get_batch_endpoint() batch_body = {} index = 0 + empty_requests = [] for row_request_parameters in batch_requests_parameters: + if row_request_parameters.get("url") is None: + empty_requests.append(index) + index += 1 + continue batch_request = {} batch_request["Method"] = method batch_request["Resource"] = "{}".format(row_request_parameters.get("url")) @@ -309,6 +325,9 @@ def _batch_requests(self, batch_requests_parameters, method=None): response = self.post_value(url=batch_endpoint, data=batch_body) json_response = simplejson.loads(response.content) for index in range(0, len(batch_requests_parameters)): + if index in empty_requests: + yield {} + continue batch_section = json_response.get("{}".format(index), {}) yield batch_section @@ -483,6 +502,33 @@ def get_item_from_url(self, url): ) return json_response + def get_next_item_from_url(self, url, params = None): + headers = self.get_requests_headers() + params = params or {} + while url: + json_response = self.get( + url=url, + headers=headers, + params=params, + can_raise=False, + error_source="get_next_item_from_url" + ) + next_url = get_next_page_url(json_response) + if next_url != url: + url = next_url + else: + # Some endpoints lead to a loop + url = None + if isinstance(json_response, list): + for item in json_response: + yield item + elif "Items" in json_response: + items = json_response.get("Items", []) + for item in items: + yield item + else: + yield json_response + def get(self, url, headers, params, can_raise=True, error_source=None): error_message = None url = build_query_string(url, params) @@ -697,6 +743,8 @@ def search_attributes(self, database_webid, **kwargs): "query": query, "databaseWebId": database_webid } + if "search_associations" in kwargs: + params["associations"] = kwargs.get("search_associations") json_response = self.get(url=search_attributes_base_url, headers=headers, params=params) if OSIsoftConstants.DKU_ERROR_KEY in json_response: yield json_response @@ -710,6 +758,122 @@ def search_attributes(self, database_webid, **kwargs): else: json_response = None + def search_elements(self, database_webid, name=None, description=None, category=None, template=None, full_search=True): + headers = self.get_requests_headers() + tempo_maxcount = OSIsoftConstants.DEFAULT_MAXCOUNT + params = { + "maxCount": tempo_maxcount, + "associations": "Paths", + } + url = self.endpoint.get_base_url() + "/assetdatabases/{}/elements".format(database_webid) + if name: + params["nameFilter"] = name + if description: + params["descriptionFilter"] = description + if category: + params["categoryName"] = category + if template: + params["templateName"] = template + if full_search: + params["searchFullHierarchy"] = True + json_response = self.get(url=url, headers=headers, params=params) + if OSIsoftConstants.DKU_ERROR_KEY in json_response: + yield json_response + start_index = 0 + while json_response: + items = json_response.get(OSIsoftConstants.API_ITEM_KEY, []) + for item in items: + yield item + if len(items) < tempo_maxcount: + logger.info("No more result items") + return + start_index += tempo_maxcount + logger.info("Trying again with startIndex={}".format(start_index)) + params["startIndex"] = start_index + json_response = self.get(url=url, headers=headers, params=params) + + def batched_search(self, database, element_name, attribute_name, element_category, + attribute_category, template, restrict_to_elements, + elements_max_count=None, attributes_max_count=None): + elements_query = { + "templateName": template, + "categoryName": element_category, + "nameFilter": element_name, + "searchFullHierarchy": "true", + "associations": "Paths" + } + if elements_max_count: + elements_query["maxCount"] = elements_max_count + attribute_query = { + "searchFullHierarchy": "true", + "associations": "Paths" + } + if attribute_name: + attribute_query["nameFilter"] = attribute_name + if attribute_category: + attribute_query["categoryName"] = attribute_category + if attributes_max_count: + attribute_query["maxCount"] = attributes_max_count + elements_url = "{}/elements".format(database) + if not restrict_to_elements: + request_body = { + "elements": { + "Method": "GET", + "Resource": "{}{}".format( + elements_url, + build_query_string("", elements_query) + ) + }, + "attributes": { + "Method": "GET", + "RequestTemplate": { + "Resource": "{{0}}{}".format( + build_query_string("", attribute_query) + ) + }, + "ParentIds": ["elements"], + "Parameters": ["$.elements.Content.Items[*].Links.Attributes"] + } + } + url = self.endpoint.get_batch_endpoint() + headers = OSIsoftConstants.WRITE_HEADERS + response = self.post(url, headers=headers, data=request_body, params={}) + json_response = response.json() + attributes = json_response.get("attributes", {}) + attributes_content = attributes.get("Content", {}) + if not isinstance(attributes_content, dict): + # the search returned nothing + return + attributes_content_items = attributes_content.get("Items", []) + for attributes_content_item in attributes_content_items: + content = attributes_content_item.get("Content", {}) + sub_items = content.get("Items", []) + for sub_item in sub_items: + yield sub_item + else: + count = 1 + request_body = {} + for restrict_to_element in restrict_to_elements: + job_tag = "J_{}".format(count) + request_body[job_tag] = { + "Method": "GET", + "Resource": "{}/attributes{}".format( + restrict_to_element, + build_query_string("", attribute_query) + ) + } + count += 1 + url = self.endpoint.get_batch_endpoint() + headers = OSIsoftConstants.WRITE_HEADERS + response = self.post(url, headers=headers, data=request_body, params={}) + json_response = response.json() + for job_tag in json_response: + job_result = json_response.get(job_tag) + content = job_result.get("Content", {}) + sub_items = content.get("Items", []) + for sub_item in sub_items: + yield sub_item + def build_element_query(self, **kwargs): element_query_keys = { "element_name": "Name:'{}'", @@ -775,9 +939,59 @@ def traverse(self, path_elements): json_response = self.get(url=next_url, headers=headers, params={}, error_source="traverse") if attribute: item = self.extract_item_with_name(json_response, attribute) - return item + def traverse_and_cache(self, ex_path_elements, ex_path_attributes, tree, trim_siblings=True): + path_elements = ex_path_elements.copy() + path_attributes = ex_path_attributes.copy() + full_path_elements = path_elements.copy() + path_attributes.copy() + if tree.exists(full_path_elements): + sending_back = tree.get(full_path_elements) + return sending_back + if path_attributes: + attribute_to_search = path_attributes.pop() + cached_item = self.traverse_and_cache(path_elements, path_attributes, tree, trim_siblings=trim_siblings) + last_know_url = cached_item.get("url") + "/attributes" + headers = self.get_requests_headers() + json_response = self.get(url=last_know_url, headers=headers, params={}, error_source="recursive traverse_and_cache") + for item in json_response.get(OSIsoftConstants.API_ITEM_KEY, []): + item_name = item.get("Name") + tree.put(path_elements + path_attributes + [item_name], get_item_details(item)) + item = self.extract_item_with_name(json_response, attribute_to_search) + return get_item_details(item) + + element_to_search = path_elements.pop() + cached_item = self.traverse_and_cache(path_elements, [], tree, trim_siblings=trim_siblings) + last_know_url = cached_item.get("url") + "/elements" + headers = self.get_requests_headers() + json_response = self.get(url=last_know_url, headers=headers, params={}, error_source="recursive traverse_and_cache") + for item in json_response.get(OSIsoftConstants.API_ITEM_KEY, []): + item_name = item.get("Name") + if not trim_siblings or item_name in ex_path_elements: + tree.put(path_elements + [item_name], get_item_details(item)) + item = self.extract_item_with_name(json_response, element_to_search) + return get_item_details(item) + + def get_attributes_templates_names(self, templates_urls): + batch_requests_parameters = [] + templates_names = [] + for template_url in templates_urls: + request_kwargs = { + "url": template_url, + "headers": self.get_requests_headers() + } + batch_requests_parameters.append(request_kwargs) + json_responses = self._batch_requests(batch_requests_parameters) + for json_response in json_responses: + response_content = json_response.get("Content", {}) + template_path = response_content.get("Path", "") + template_name_match = re.search(r'ElementTemplates\[([^\]]+)\]', template_path) + template_name = None + if template_name_match: + template_name = template_name_match.group(1) + templates_names.append(template_name) + return templates_names + def split_element_attribute(self, path_element): attribute = None path_elements = path_element.split("|") @@ -993,7 +1207,7 @@ def close(self): def validate_timestamp(timestamp): - valid_formats=["%Y-%m-%dT%H:%M:%S.%fZ", "%Y-%m-%dT%H:%M:%SZ"] + valid_formats = ["%Y-%m-%dT%H:%M:%S.%fZ", "%Y-%m-%dT%H:%M:%SZ"] for valid_format in valid_formats: try: datetime.strptime(timestamp, valid_format) @@ -1017,7 +1231,7 @@ def build_query_string(url, params): if isinstance(value, list): for element in value: tokens.append(key+"="+str(element)) - else: + elif value is not None: tokens.append(key+"="+str(value)) if len(tokens) > 0: return url + "?" + "&".join(tokens) diff --git a/python-lib/osisoft_constants.py b/python-lib/osisoft_constants.py index 3afbbd0d..ff6854b4 100644 --- a/python-lib/osisoft_constants.py +++ b/python-lib/osisoft_constants.py @@ -405,7 +405,7 @@ class OSIsoftConstants(object): "Security": "{base_url}/eventframes/{webid}/security", "SecurityEntries": "{base_url}/eventframes/{webid}/securityentries" } - PLUGIN_VERSION = "1.4.0" + PLUGIN_VERSION = "1.4.2-beta.1" VALUE_COLUMN_SUFFIX = "_val" WEB_API_PATH = "piwebapi" WRITE_HEADERS = {'X-Requested-With': 'XmlHttpRequest'} diff --git a/python-lib/osisoft_plugin_common.py b/python-lib/osisoft_plugin_common.py index 2a160dda..5a8570ab 100644 --- a/python-lib/osisoft_plugin_common.py +++ b/python-lib/osisoft_plugin_common.py @@ -56,6 +56,21 @@ def get_credentials(config, can_raise=True): return auth_type, username, password, server_url, is_ssl_check_disabled, error_message +def get_batch_parameters(config): + credentials = config.get("credentials", {}) + max_request_size = credentials.get("max_request_size", 1000) + estimated_density = credentials.get("estimated_density", 6000) + maximum_points_returned = credentials.get("maximum_points_returned", 1000000) + return max_request_size, estimated_density, maximum_points_returned + + +def compute_time_spent(start, end, bla): + # 2023-06-30T13:05:10.8692786Z->2024-06-30T13:05:10.9640942Z + start = iso_to_epoch(start) + end = iso_to_epoch(end) + return end - start + + def get_advanced_parameters(config): show_advanced_parameters = config.get('show_advanced_parameters', False) batch_size = 500 @@ -139,6 +154,7 @@ def build_requests_params(**kwargs): "boundary_type": "syncTimeBoundaryType", "name_filter": "nameFilter", "category_name": "categoryName", + "description": "descriptionFilter", "template_name": "templateName", "referenced_element_name_filter": "referencedElementNameFilter", "referenced_element_template": "referencedElementTemplate", @@ -427,9 +443,9 @@ def epoch_to_iso(epoch): def iso_to_epoch(iso_timestamp): - logger.info("Converting iso timestamp '{}' to epoch".format(iso_timestamp)) + # logger.info("Converting iso timestamp '{}' to epoch".format(iso_timestamp)) if is_epoch(iso_timestamp): - logger.info("Timestamp is already epoch") + # logger.info("Timestamp is already epoch") return iso_timestamp epoch_timestamp = None try: @@ -438,7 +454,7 @@ def iso_to_epoch(iso_timestamp): except Exception: logger.error("Error when converting iso timestamp '{}' to epoch".format(iso_timestamp)) return None - logger.info("Timestamp is now '{}'".format(epoch_timestamp)) + # logger.info("Timestamp is now '{}'".format(epoch_timestamp)) return epoch_timestamp @@ -481,7 +497,7 @@ def fields_selector(data_type): def get_next_page_url(json): - if not json: + if not isinstance(json, dict): return None next_page_url = json.get("Links", {}).get("Next", "").replace('&', '&') if next_page_url: @@ -600,3 +616,140 @@ def get_worst_performers(self): for slowest_event, slowest_time in zip(self.slowest_events, self.slowest_times): worst_performers.append("{}: {}s".format(slowest_event, slowest_time)) return worst_performers + + +class BatchTimeCounter(object): + def __init__(self, max_time_to_retrieve_per_batch): + logger.info("BatchTimeCounter:max_time_to_retrieve_per_batch={}s".format(max_time_to_retrieve_per_batch * 60 * 60)) + self.max_time_to_retrieve_per_batch = max_time_to_retrieve_per_batch * 60 * 60 + self.total_batched_time = 0 + + def is_batch_full(self): + if self.max_time_to_retrieve_per_batch < 0: + return False + if self.total_batched_time > self.max_time_to_retrieve_per_batch: + logger.warning("batch contains {}s of request, needs to flush now".format(self.total_batched_time)) + self.total_batched_time = 0 + return True + return False + + def add(self, start_time, end_time, interval): + self.total_batched_time += compute_time_spent(start_time, end_time, interval) + + +def get_item_details(item): + KEYS_TO_CHECK = { + "Name": "title", "TemplateName": "template_name", "CategoryNames": "category_names", "Description": "description", + "HasChildren": "has_children", "Path": "path", "Paths": "paths", "WebId": "id", "checked": "checked", "BaseTemplate": "BaseTemplate" + } # should we stick to python naming convention or keep pi's ones throughout ? + details = {} + for key_to_check in KEYS_TO_CHECK: + value = item.get(key_to_check) + if value: + details[KEYS_TO_CHECK.get(key_to_check)] = value + details["url"] = item.get("Links", {}).get("Self") + details["type"] = "attribute" if "|" in details.get("path", "") else "element" + return details + + +class Tree(): + # Each put + # - stores the data in the index + # - builds a tree based on the data's path, pointing at the right index + def __init__(self, root_tree=None): + self.tree = {} + self.index = [] + if root_tree: + self._ingest(root_tree) + + def _ingest(self, root_tree, parent_path=None): + parent_path = parent_path or [] + if isinstance(root_tree, list): + for item in root_tree: + if not parent_path: + path = item.get("path", "") + parent_path = path.split("\\")[2:][0:2] + item_children = item.pop("children", []) + title = item.get("title") + self._ingest(item_children, parent_path=parent_path + [title]) + path = item.get("path", "") + self.put(parent_path + [title], item) + + def put(self, path, data): + if isinstance(path, list): + current_level = self.tree + for token in path: + if token not in current_level: + current_level[token] = {} + current_level = current_level.get(token) + index_to_update = current_level.get("_v", None) + if index_to_update is not None: + self.index[index_to_update] = data + else: + last_index = len(self.index) + self.index.append(data) + current_level.update({"_v": last_index}) + + def get(self, path, default=None): + if isinstance(path, list): + current_level = self.tree + for token in path: + if token not in current_level: + return default + else: + current_level = current_level.get(token) + index = current_level.get("_v") + return self.get_record(index) + + def get_tree(self): + return self.tree + + def get_record(self, index): + if index < len(self.index): + return self.index[index] + return None + + def get_records(self): + return self.index + + def exists(self, path): + current = self.tree + if isinstance(path, list): + for token in path: + current = current.get(token, {}) + if not current: + return False + return True + return False + + def print(self): + print("Tree {}".format(self.tree)) + print("Tree content {}".format(self.index)) + + +def recursive_tree_rebuild(dictionary, records, counter=None): + counter = counter or -1 + output = [] + + for key in dictionary: + if key == "_v": + continue + sub_dictionary = dictionary.get(key) + context = {} + if "_v" in sub_dictionary: + index_id = sub_dictionary.get("_v") + if isinstance(index_id, int): + context = records[index_id] + counter += 1 + if sub_dictionary: + counter += 1 + children = recursive_tree_rebuild(sub_dictionary, records, counter + 1) + else: + children = [] + # context["id"] = str(counter) + context["title"] = key + context["expanded"] = True + # context["checked"] = False + context["children"] = children + output.append(context) + return output diff --git a/resource/attribute-table-row.html b/resource/attribute-table-row.html new file mode 100644 index 00000000..c480ff30 --- /dev/null +++ b/resource/attribute-table-row.html @@ -0,0 +1,40 @@ + + + +{{ctrl.mergedAttribute.title}} +{{ctrl.mergedAttribute.description}} + + + {{path}}
+ + + + + + +
+ {{aggregate.label}} + + + + +
+ diff --git a/resource/browse_af_tree.py b/resource/browse_af_tree.py new file mode 100644 index 00000000..bea164a1 --- /dev/null +++ b/resource/browse_af_tree.py @@ -0,0 +1,476 @@ +from osisoft_client import OSIsoftClient +from safe_logger import SafeLogger +from osisoft_plugin_common import get_credentials, build_select_choices, check_debug_mode +from osisoft_plugin_common import get_item_details, Tree, recursive_tree_rebuild, PerformanceTimer +import time + +logger = SafeLogger("PI System plugin", ["user", "password"]) + + +def _run_and_log_login_call(method_name, api_call): + started_at = time.time() + try: + response = api_call() + choices = response.get("choices") if isinstance(response, dict) else None + choice_count = len(choices) if isinstance(choices, list) else "n/a" + preview_titles = [] + if isinstance(choices, list): + for choice in choices[:3]: + if isinstance(choice, dict): + preview_titles.append(choice.get("title", "")) + else: + preview_titles.append(str(choice)) + logger.info( + "[LOGIN] {} ok | {}ms | choices_count={} | preview_titles={}".format( + method_name, + int((time.time() - started_at) * 1000), + choice_count, + preview_titles + ) + ) + return response + except Exception as error: + logger.error( + "[LOGIN] {} failed | {}ms | error={}".format( + method_name, + int((time.time() - started_at) * 1000), + error + ) + ) + raise + + +def do(payload, config, plugin_config, inputs): + if "config" in config: + config = config.get("config") + if "credentials" not in config: + return {"choices": [{"label": "Requires DSS v10.0.4 or above. Please use the OSIsoft Search custom dataset instead"}]} + elif config.get("credentials") == {}: + return {"choices": [{"label": "Pick a credential"}]} + + auth_type, username, password, server_url, is_ssl_check_disabled, credential_error = get_credentials(config, can_raise=False) + is_ssl_check_disabled = config.get("is_ssl_check_disabled", False) # Because no advanced parameter switch + + if credential_error: + return build_select_choices(credential_error) + + if not (auth_type and username and password): + return build_select_choices("Pick a credential") + + if not username or not password: + return build_select_choices( + "Incorrect credential. " + + "Go to you profile page > Credentials > Your preset, click the edit button and fill in you username and password details." + ) + + if not server_url: + return build_select_choices("Fill in the server address") + + is_debug_mode = check_debug_mode(config) + + network_timer = PerformanceTimer() + + client = OSIsoftClient( + server_url, auth_type, username, password, + is_ssl_check_disabled=is_ssl_check_disabled, is_debug_mode=is_debug_mode, + network_timer=network_timer + ) + + method = payload.get("method") + if method == "get_query_catalogs": + return get_query_catalogs(None, config) + if method == "get_children_from_db": + database_name = config.get("database_name") + parent = payload.get("parent", {}) + return _run_and_log_login_call( + "get_children_from_db", + lambda: get_children_from_db(client, parent, database_name=database_name) + ) + if method == "get_templates_from_db": + database_name = config.get("database_name") + parent = payload.get("parent", {}) + return _run_and_log_login_call( + "get_templates_from_db", + lambda: get_template_hierarchy_from_db(client, parent, database_name=database_name) + ) + if method == "get_attribute_categories_from_db": + database_name = config.get("database_name") + parent = payload.get("parent", {}) + return _run_and_log_login_call( + "get_attribute_categories_from_db", + lambda: get_items_from_db(client, parent, "AttributeCategories", database_name=database_name) + ) + if method == "get_element_categories_from_db": + database_name = config.get("database_name") + parent = payload.get("parent", {}) + return _run_and_log_login_call( + "get_element_categories_from_db", + lambda: get_items_from_db(client, parent, "ElementCategories", database_name=database_name) + ) + if method == "do_search": + template_name = config.get("template", None) + category_name = config.get("element_category", None) + clicked_nodes = config.get("clickedNodes", []) + if not isinstance(clicked_nodes, list): + clicked_nodes = [] + active_tab = config.get("activeTab") + selected_template_names = config.get("selectedTemplateNames", []) + if template_name == "-- Any --": + template_name = None + if not isinstance(selected_template_names, list): + selected_template_names = [] + selected_template_names = [ + template_name_item for template_name_item in selected_template_names + if isinstance(template_name_item, str) and template_name_item and template_name_item != "-- Any --" + ] + if category_name == "-- Any --": + category_name = None + element_category = config.get("element_category", None) + if element_category == "-- Any --": + element_category = None + attribute_category = config.get("attribute_category", None) + if attribute_category == "-- Any --": + attribute_category = None + database_name = config.get("database_name") + element_name = config.get("element_name") + attribute_name = config.get("attribute_name") + if isinstance(element_name, str): + element_name = element_name.strip() + if element_name == "": + element_name = None + if isinstance(attribute_name, str): + attribute_name = attribute_name.strip() + if attribute_name == "": + attribute_name = None + + has_attribute_filter = attribute_name is not None + is_template_tab = active_tab == "template" + has_clicked_element_nodes = len(clicked_nodes) > 0 + # clicked_nodes scope is only for element-node URLs (batched_search restrict_to_elements). + # Template-node selections are scoped via selected_template_names. + use_clicked_element_nodes_scope = ( + has_attribute_filter and + not is_template_tab and + has_clicked_element_nodes + ) + if has_attribute_filter and not is_template_tab: + # Attribute search scope in element tab: + # - with selected nodes => restrict to selected nodes + # - without selected nodes => global search on the full tree + # In both cases, element text input is not a scope for attributes. + element_name = None + elif has_attribute_filter and is_template_tab: + # Attribute search scope in template tab: + # - with selected template nodes => restrict to selected templates + # - without selected template nodes => global search on the full tree + # Ignore stale element/template text filters for attribute-only searches. + element_name = None + if len(selected_template_names) == 0: + template_name = None + + has_element_filter = element_name is not None + use_selected_template_names_scope = ( + is_template_tab and len(selected_template_names) > 0 + ) + + if not use_clicked_element_nodes_scope and not use_selected_template_names_scope: + clicked_nodes = [] + # root_tree = payload.get("root_tree") + root_tree = config.get("treeData", []) + root_tree = shorten_tree(root_tree) + attributes = [] + # https://dku-qa-osi.francecentral.cloudapp.azure.com/piwebapi/assetdatabases/F1RD3VEt1yTvt0ip6-a5yeEVsgbMcrwu_Je0qg9btcZIvPswT1NJU09GVC1QSS1TRVJWXFdFTEw + database_webid = database_name.split("/")[-1] + elements_max_count, attributes_max_count = get_max_counts(config) + + attributes = [] + if use_selected_template_names_scope: + # In template tab with selected template nodes, scope searches to all selected templates. + # We ignore element_name here to avoid stale "*" from single-template click behavior. + for selected_template_name in selected_template_names: + for result in client.batched_search( + database_name, None, attribute_name, + element_category, attribute_category, selected_template_name, [], + elements_max_count=elements_max_count, attributes_max_count=attributes_max_count + ): + attributes.append(result) + else: + for result in client.batched_search(database_name, element_name, attribute_name, + element_category, attribute_category, template_name, clicked_nodes, + elements_max_count=elements_max_count, attributes_max_count=attributes_max_count): + # result["checked"] = True + attributes.append(result) + attributes = split_real_from_linked_paths(attributes) + items = [] + for attribute in attributes: + item = get_item_details(attribute) + items.append(item) + items = expand_items_by_paths(items) + attributesCopy = [dict(item) for item in items] + rebuilt_tree = rebuild_tree(client, items.copy(), root_tree) + logger.info("Search network timer:{}".format(network_timer.get_report())) + return {"choices": rebuilt_tree, "attributes": attributesCopy} + + parameter_name = payload.get("parameterName") + + if parameter_name == "server_name": + choices = [] + servers = client.get_asset_servers(can_raise=False) + choices.extend(servers) + return build_select_choices(choices) + + if parameter_name == "data_server_url": + choices = [] + choices.extend(client.get_data_servers(can_raise=False)) + return build_select_choices(choices) + + if parameter_name == "database_name": + choices = [] + next_url = config.get("server_name") + if next_url: + choices.extend(client.get_next_choices(next_url, "Self")) + return build_select_choices(choices) + else: + return build_select_choices() + if parameter_name == "treeData": + return {"choices": config.get("treeData")} + + return build_select_choices() + + +def get_query_catalogs(cnx, config): + user = config.get("credentials", {}).get("osisoft_basic", {}).get("user") + password = config.get("credentials", {}).get("osisoft_basic", {}).get("password") + return {"choices": [user, password]} + + +def get_items_from_db(client, parent_node, link_key, database_name=None): + default_choice = {"title": "-- Any --"} + if isinstance(parent_node, dict): + url = parent_node.get("url", database_name) + else: + url = parent_node + this_node = next(client.get_next_item_from_url(url)) + links = this_node.get("Links", {}) + items_url = links.get(link_key) + items = [] + items.append(default_choice) + if items_url: + for item in client.get_next_item_from_url(items_url): + item = get_item_details(item) + item["type"] = link_key + items.append(item) + return {"choices": items} + + +def get_children_from_db(client, parent_node, database_name=None): + if isinstance(parent_node, dict): + url = parent_node.get("url", database_name) + else: + url = parent_node + this_node = next(client.get_next_item_from_url(url, params={"associations": "Paths"})) + links = this_node.get("Links", {}) + attributes_url = links.get("Attributes") + elements_url = links.get("Elements") + children = [] + if elements_url: + elements = client.get_next_item_from_url(elements_url, params={"associations": "Paths"}) + for element in elements: + child = get_item_details(element) + # child["title"] = "🧩{}".format(child.get("title")) + child["type"] = "element" + child["children"] = [] + children.append(child) + if attributes_url: + attributes = client.get_next_item_from_url(attributes_url, params={"associations": "Paths"}) + templates_urls = [] + for attribute in attributes: + # templates_urls are processed in batch for speed + templates_urls.append(extract_attribute_template_url(attribute)) + child = get_item_details(attribute) + # child["title"] = "🏷️{}".format(child.get("title")) + child["type"] = "attribute" + if child.get("has_children"): + child["children"] = [] + children.append(child) + templates_names = client.get_attributes_templates_names(templates_urls) + # post processing the batch response + for child, template_name in zip(children, templates_names): + if template_name: + child["template_name"] = template_name + return {"choices": children} + + +def extract_attribute_template_url(attribute): + return attribute.get("Links", {}).get("Template") + + +def get_template_hierarchy_from_db(client, parent_node, database_name=None): + if isinstance(parent_node, dict): + url = parent_node.get("url", database_name) + else: + url = parent_node + default_choice = {"title": "-- Any --", "id:": ""} + this_node = next(client.get_next_item_from_url(url)) + links = this_node.get("Links", {}) + element_templates_url = links.get("ElementTemplates") + children = [default_choice] + rebuilt_tree = [] + if element_templates_url: + element_templates = client.get_next_item_from_url(element_templates_url) + for element_template in element_templates: + child = get_item_details(element_template) + child["type"] = "template" + child["children"] = [] + children.append(child) + rebuilt_tree = nest_children(children) + return {"choices": rebuilt_tree} + + +def nest_children(items): + name_to_item = {item["title"]: item for item in items} + tree = [] + for item in items: + parent_name = item.get("BaseTemplate") + if parent_name is None or parent_name not in name_to_item: + tree.append(item) + else: + parent = name_to_item[parent_name] + if "children" not in parent: + parent["children"] = [] + parent["children"].append(item) + parent["has_children"] = True + return tree + + +def rebuild_tree(client, items, root_tree=None): + # builds an active tree containing all the items and their parent up to the root + tree = Tree(root_tree=root_tree) + # tree.print() + while items: + item = items.pop() + if item is None: + break + find_all_ancestors(client, item, tree) + update_item(item, tree) + result = recursive_tree_rebuild(tree.get_tree(), tree.get_records()) + result = drop_first_levels(result) + return result + + +def drop_first_levels(result): + # recursively removes the 2 first levels of the returned tree + # (server and DB) + output_result = [] + for item in result: + path = item.get("path", "") + path_length = len(path.split("\\")) + if path_length >= 5: + output_result.append(item) + else: + children = item.get("children", []) + output_result = drop_first_levels(children) + return output_result + + +def find_all_ancestors(client, item, tree): + # Find all the ancestors of an item + elements_paths_tokens, attributes_paths_tokens = path_to_list(item.get("path")) + client.traverse_and_cache(elements_paths_tokens, attributes_paths_tokens, tree) + + +def combine_trees(final_tree, all_item_s_ancestors): + # combine two trees with partial overlap and common root ancestor + return final_tree + + +# elements, attributes +def path_to_list(path): + if not path: + return [], [] + return path.split('|')[0].split('\\')[2:], (path.split('|')[1:]) + + +def shorten_tree(tree): + if isinstance(tree, list): + for node in tree: + if "expanded" in node: + # node.pop("expanded", None) + node["expanded"] = False + if "children" in node: + shorten_tree(node.get("children", [])) + return tree + + +def split_real_from_linked_paths(attributes): + for attribute in attributes: + current_path = attribute.get("path", attribute.get("Path")) + paths = attribute.get("Paths") + if isinstance(paths, list): + attribute["Paths"] = [path for path in paths if path != current_path] + return attributes + + +def expand_items_by_paths(items): + expanded_items = [] + seen = set() + for item in items: + if not isinstance(item, dict): + continue + current_path = item.get("path") + linked_paths = item.get("paths", []) + candidate_paths = [] + if current_path: + candidate_paths.append(current_path) + if isinstance(linked_paths, list): + for linked_path in linked_paths: + if linked_path and linked_path not in candidate_paths: + candidate_paths.append(linked_path) + if not candidate_paths: + candidate_paths = [None] + for candidate_path in candidate_paths: + expanded_item = dict(item) + if candidate_path: + expanded_item["path"] = candidate_path + if isinstance(linked_paths, list): + expanded_item["paths"] = [path for path in candidate_paths if path and path != candidate_path] + dedupe_key = (expanded_item.get("id"), expanded_item.get("path"), expanded_item.get("title")) + if dedupe_key in seen: + continue + seen.add(dedupe_key) + expanded_items.append(expanded_item) + return expanded_items + + +def set_as_selected(items): + for item in items: + item["checked"] = True + return items + + +def update_item(item, tree): + elements_paths_tokens, attributes_paths_tokens = path_to_list(item.get("path")) + if not elements_paths_tokens and not attributes_paths_tokens: + return + tree.put(elements_paths_tokens + attributes_paths_tokens, item) + + +def get_max_counts(config): + show_advanced_parameters = config.get("show_advanced_parameters", False) + if not show_advanced_parameters: + return 100, 100 + + def parse_max_count(value, default): + if value is None or value == "": + return default + try: + value = int(value) + except (TypeError, ValueError): + return default + if value <= 0: + return None + return value + + elements_max_count = parse_max_count(config.get("elements_max_count"), 100) + attributes_max_count = parse_max_count(config.get("attributes_max_count"), 100) + return elements_max_count, attributes_max_count diff --git a/resource/pi-system_af-explorer.css b/resource/pi-system_af-explorer.css new file mode 100644 index 00000000..d950cdc0 --- /dev/null +++ b/resource/pi-system_af-explorer.css @@ -0,0 +1,262 @@ +.fh.w800.oa { + width: 100% !important; +} + +.tree-node__label--search-result{ + background-color: yellow; +} + +.tree-node__label--clickable { + box-shadow: inset 0 0 0 1px #2e7d32; + border-radius: 2px; +} + +.tree-node__label--clickable::after { + content: ""; + display: inline-block; + width: 6px; + height: 6px; + margin-left: 6px; + border-radius: 50%; + background-color: #2e7d32; + vertical-align: middle; +} +.generic-class { + display: flex; + align-items: flex-start; + flex: 1 0 0; + align-self: stretch; + flex-direction: row-reverse; +} +.pi-system-explorer__main { + display: flex; + padding: 16px; + flex-direction: column; + align-items: flex-start; + gap: 16px; + flex: 1 0 0; + align-self: stretch; +} +.pi-system-explorer__tree-view-container { + display: flex; + width: 252px; + padding: 16px 12px; + flex-direction: column; + align-items: flex-start; + gap: 12px; + align-self: stretch; + border-right: 1px solid #BBB; + background: #FFF; +} + +.pi-system-explorer__authentication { + display: flex; + flex-direction: column; + align-items: flex-start; + align-self: stretch; +} +.pi-system-explorer__authentication-header { + display: flex; + height: 32px; + padding: 0 12px; + justify-content: space-between; + align-items: center; + align-self: stretch; + border: 1px solid #999; + background: linear-gradient(0deg, rgba(242, 242,242,0.50) 100%), #FFFFFF; +} + +.pi-system-explorer__authentication-body { + display: flex; + padding: 16px 12px; + flex-direction: column; + justify-content: center; + align-items: flex-start; + gap: 16px; + align-self: stretch; + border-right: 1px solid var(--Greyscale-grey-lighten-4, #999); + border-bottom: 1px solid var(--Greyscale-grey-lighten-4, #999); + border-left: 1px solid var(--Greyscale-grey-lighten-4, #999); + background: #FFF; +} + +.pi-system-explorer__authentication__submit { + display: flex; + flex-direction: column; + align-items: flex-end; + gap: 8px; + align-self: stretch; +} + +.pi-system-config--toggle-advanced, .pi-system-config--ssl-check { + display: flex; + padding-bottom: 4px; + align-items: flex-start; + gap: 8px; + width: 220px; +} + +.pi-system-config--params-advanced { + display: flex; + gap: 64px; +} + +.pi-system-config--mandatory { + display: flex; + gap: 64px; +} + +.attribute-section__search { + display: flex; + padding: 16px; + flex-direction: column; + align-items: flex-start; + gap: 16px; + flex: 1 0 0; + align-self: stretch; +} + +.custom-table { + border-collapse: collapse; + width: 100%; + font-family: Arial, sans-serif; +} + +.custom-table th, +.custom-table td { + border: 1px solid #ccc; + padding: 8px 12px; + text-align: left; +} + +.custom-table th { + background-color: #f4f4f4; + font-weight: bold; +} + +.custom-table tr:nth-child(even) { + background-color: #fafafa; +} + +.custom-table tr:hover { + background-color: #f1f7ff; +} + +.custom-table tr.custom-table__group-row td { + background-color: #e8f0fb; + font-weight: 600; +} + +.custom-table tr.custom-table__group-row:hover td { + background-color: #e8f0fb; +} + +.custom-table tr.custom-table__empty-row td { + color: #666; + font-style: italic; +} + +.attribute-table-block { + width: 100%; + margin-bottom: 14px; +} + +.attribute-table-block__title { + margin-bottom: 6px; +} + +.tree-view__tabs{ + display: flex; + align-items: center; + align-self: stretch; + border: 1px solid #BBB; +} + +.tree-view__tab { + display: flex; + align-items: flex-start; + flex: 1 0 0; + border-top: 1px solid #BBB; + border-right: 0.5px solid #BBB; + border-bottom: 1px solid #BBB; + border-left: 1px solid #BBB; + &:hover, &:active { + background-color: #E7F3FF; + } +} + +.tree-view__tab-content { + display: flex; + padding: 12px 23px; + justify-content: center; + align-items: center; + gap: 4px; + flex: 1 0 0; + color: #000; + font-family: "Source Sans Pro"; + font-size: 14px; + font-style: normal; + font-weight: 600; + line-height: normal; +} + +.tree-view__content { + gap: 8px; + display: flex; + flex-direction: column; +} + +.pi-system-explorer__tree-view-container{ + .pi-system-explorer__tree-view { + .tree-view__search-container{ + display:flex; + align-items:center; + gap: 8px; + align-self: stretch; + } + .tree-view__search-box { + display: flex; + height: 26px; + padding: 0 8px; + flex-direction: column; + justify-content: center; + align-items: flex-start; + flex: 1 0 0; + } + } +} + +.tab-container { + font-family: Arial, sans-serif; +} + +.tab-header { + list-style: none; + padding: 0; + margin: 0 0 6px 0; + display: flex; + border-bottom: 2px solid #ddd; +} + +.tab-header li { + padding: 8px 16px; + cursor: pointer; + background: #f5f5f5; + border: 1px solid #ddd; + border-bottom: none; + margin-right: 4px; +} + +.tab-header li.active { + background: #fff; + font-weight: bold; +} + +.tab-content { + border-top: none; + padding: 10px; +} + +.recipe-editor-page .formbased-recipe-infozone h1, .recipe-editor-page .recipe-settings-section1 h1, .recipe-editor-page .formbased-recipe-infozone .recipe-settings-section2, .recipe-editor-page .recipe-settings-section1 .recipe-settings-section2{ + padding: 0px !important; +} diff --git a/resource/pi-system_af-explorer.html b/resource/pi-system_af-explorer.html new file mode 100644 index 00000000..b01977b0 --- /dev/null +++ b/resource/pi-system_af-explorer.html @@ -0,0 +1,204 @@ + +
+
+
+
+ Welcome to Pi System Plugin +
+
+
+
+ +
+
+
+
+
+
+ +
+ +
+
+
+ +
+ +
+
+
+
+ + + Advanced parameters +
+
+ +
Disable SSL Check +
+
+
Elements max count
+
+
+
+
Attributes max count
+
+
+
+
+ +
+
+
+
+ +
+ +
+
+ +
+
+
Attributes and templates
+
+ + + +
+ Filter by category + + + Title + Description + Path + Data type + Aggregates + + + + + {{section.emptyMessage}} + + + + + + + + + {{group.group_name}} + + + + + +
+
+
+
+
+
+
+
Element
+
+
+
Template
+
+
+
+ + +
+ +
+
+ Filter by category