feat: pipeline_tree静态检查增强 --story=130810130

bkflow/apigw/serializers/task.py

@@ -25,9 +25,9 @@
 from pipeline.exceptions import PipelineException
 from rest_framework import serializers
 
-from bkflow.constants import MAX_LEN_OF_TASK_NAME, USER_NAME_MAX_LENGTH
+from bkflow.constants import MAX_LEN_OF_TASK_NAME, USER_NAME_MAX_LENGTH, ValidateType
 from bkflow.label.models import Label
-from bkflow.pipeline_web.parser.validator import validate_web_pipeline_tree
+from bkflow.pipeline_validate.handler import ValidatorHandler
 from bkflow.template.models import TemplateMockData
 from bkflow.utils.strings import standardize_pipeline_node_name
 
@@ -199,7 +199,7 @@ class PipelineTreeSerializer(serializers.Serializer):
     def validate_pipeline_tree(self, pipeline_tree):
         try:
             standardize_pipeline_node_name(pipeline_tree)
-            validate_web_pipeline_tree(pipeline_tree)
+            ValidatorHandler.validate(pipeline_tree, validate_type=ValidateType.TASK)
         except PipelineException as e:
             raise serializers.ValidationError(_("流程树校验失败: {}").format(str(e)))
 

bkflow/apigw/serializers/template.py

@@ -20,11 +20,15 @@
 
 from django.db.models import Q
 from django.utils.translation import ugettext_lazy as _
-from pipeline.validators import validate_pipeline_tree
 from rest_framework import serializers
 
-from bkflow.constants import MAX_LEN_OF_TEMPLATE_NAME, USER_NAME_MAX_LENGTH
+from bkflow.constants import (
+    MAX_LEN_OF_TEMPLATE_NAME,
+    USER_NAME_MAX_LENGTH,
+    ValidateType,
+)
 from bkflow.label.models import Label
+from bkflow.pipeline_validate.handler import ValidatorHandler
 from bkflow.space.models import Space
 from bkflow.template.models import Template
 
@@ -105,7 +109,7 @@ def validate(self, attrs):
 
         if pipeline_tree:
             try:
-                validate_pipeline_tree(pipeline_tree, cycle_tolerate=True)
+                ValidatorHandler.validate(pipeline_tree, validate_type=ValidateType.TEMPLATE)
             except Exception as e:
                 logger.exception(f"CreateTemplateSerializer pipeline validate error, err = {e}")
                 raise serializers.ValidationError(_(f"参数校验失败，pipeline校验不通过, err={e}"))
@@ -166,7 +170,7 @@ def validate(self, attrs):
 
         if pipeline_tree:
             try:
-                validate_pipeline_tree(pipeline_tree, cycle_tolerate=True)
+                ValidatorHandler.validate(pipeline_tree, validate_type=ValidateType.TEMPLATE)
             except Exception as e:
                 logger.exception(f"CreateTemplateSerializer pipeline validate error, err = {e}")
                 raise serializers.ValidationError(_(f"参数校验失败，pipeline校验不通过, err={e}"))

bkflow/constants.py

@@ -155,3 +155,9 @@ class VariableType(Enum):
 
 
 VARIABLE_TYPES = [VariableType.SPACE.value, VariableType.SCOPE.value]
+
+
+class ValidateType(Enum):
+    TEMPLATE = "template"
+    TASK = "task"
+    GENERAL = "general"

bkflow/pipeline_validate/__init__.py

@@ -0,0 +1,18 @@
+"""
+TencentBlueKing is pleased to support the open source community by making
+蓝鲸流程引擎服务 (BlueKing Flow Engine Service) available.
+Copyright (C) 2024 THL A29 Limited,
+a Tencent company. All rights reserved.
+Licensed under the MIT License (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at http://opensource.org/licenses/MIT
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on
+an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
+either express or implied. See the License for the
+specific language governing permissions and limitations under the License.
+
+We undertake not to change the open source license (MIT license) applicable
+
+to the current version of the project delivered to anyone in the future.
+"""

bkflow/pipeline_validate/apps.py

@@ -0,0 +1,31 @@
+"""
+TencentBlueKing is pleased to support the open source community by making
+蓝鲸流程引擎服务 (BlueKing Flow Engine Service) available.
+Copyright (C) 2024 THL A29 Limited,
+a Tencent company. All rights reserved.
+Licensed under the MIT License (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at http://opensource.org/licenses/MIT
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on
+an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
+either express or implied. See the License for the
+specific language governing permissions and limitations under the License.
+
+We undertake not to change the open source license (MIT license) applicable
+
+to the current version of the project delivered to anyone in the future.
+"""
+
+from django.apps import AppConfig
+
+
+class PipelineValidateConfig(AppConfig):
+    default_auto_field = "django.db.models.BigAutoField"
+    name = "bkflow.pipeline_validate"
+    label = "pipeline_validate"
+    verbose_name = "流程校验"
+
+    def ready(self):
+        # 导入 validators 包，触发所有校验类的加载与注册
+        import bkflow.pipeline_validate.validators  # noqa

bkflow/pipeline_validate/handler.py

@@ -0,0 +1,58 @@
+"""
+TencentBlueKing is pleased to support the open source community by making
+蓝鲸流程引擎服务 (BlueKing Flow Engine Service) available.
+Copyright (C) 2024 THL A29 Limited,
+a Tencent company. All rights reserved.
+Licensed under the MIT License (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at http://opensource.org/licenses/MIT
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on
+an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
+either express or implied. See the License for the
+specific language governing permissions and limitations under the License.
+
+We undertake not to change the open source license (MIT license) applicable
+
+to the current version of the project delivered to anyone in the future.
+"""
+from typing import Optional
+
+from pipeline.exceptions import PipelineException
+
+from bkflow.constants import ValidateType
+
+
+class ValidatorHandler:
+    """校验器处理器"""
+
+    __hub = {}
+
+    @classmethod
+    def register(cls, validator_cls) -> None:
+        """注册校验器类"""
+        if validator_cls.name is None:
+            raise ValueError(f"校验器 {validator_cls.__name__} 的 name 属性不能为 None")
+        if validator_cls.name in cls.__hub:
+            existing_cls = cls.__hub[validator_cls.name]
+            raise ValueError(f"校验器名称 '{validator_cls.name}' 已被 {existing_cls.__name__} 注册，")
+        cls.__hub[validator_cls.name] = validator_cls
+
+    @classmethod
+    def validate(cls, web_pipeline_tree: dict, validate_type: Optional[ValidateType] = None):
+        validators_to_run = []
+        for validator_name, validator_cls in cls.__hub.items():
+            # 获取校验器的类型
+            validator_validate_type = getattr(validator_cls, "validate_type", None)
+
+            if validate_type is None:
+                # 默认行为：执行所有校验器
+                validators_to_run.append((validator_name, validator_cls))
+            elif validator_validate_type in [validate_type.value, ValidateType.GENERAL.value]:
+                # 指定类型：执行匹配类型和通用类型的校验器
+                validators_to_run.append((validator_name, validator_cls))
+
+        for validator_name, validator_cls in validators_to_run:
+            result = validator_cls.validate(web_pipeline_tree)
+            if not result.is_valid:
+                raise PipelineException(result.error)

bkflow/pipeline_validate/validators/__init__.py

@@ -0,0 +1,32 @@
+"""
+TencentBlueKing is pleased to support the open source community by making
+蓝鲸流程引擎服务 (BlueKing Flow Engine Service) available.
+Copyright (C) 2024 THL A29 Limited,
+a Tencent company. All rights reserved.
+Licensed under the MIT License (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at http://opensource.org/licenses/MIT
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on
+an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
+either express or implied. See the License for the
+specific language governing permissions and limitations under the License.
+
+We undertake not to change the open source license (MIT license) applicable
+
+to the current version of the project delivered to anyone in the future.
+"""
+
+from bkflow.pipeline_validate.validators.general import (  # noqa
+    ConstantsValidator,
+    MakoKeywordValidator,
+    PipelineTreeValidator,
+)
+from bkflow.pipeline_validate.validators.task import ContextHydrateValidator  # noqa
+from bkflow.pipeline_validate.validators.template import (  # noqa
+    ConstantsKeyPatternValidator,
+    ConstantsSourceInfoValidator,
+    MutualExclusionValidator,
+    OutputsKeyPatternValidator,
+    SchemaValidator,
+)

bkflow/pipeline_validate/validators/base.py

@@ -0,0 +1,58 @@
+"""
+TencentBlueKing is pleased to support the open source community by making
+蓝鲸流程引擎服务 (BlueKing Flow Engine Service) available.
+Copyright (C) 2024 THL A29 Limited,
+a Tencent company. All rights reserved.
+Licensed under the MIT License (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at http://opensource.org/licenses/MIT
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on
+an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
+either express or implied. See the License for the
+specific language governing permissions and limitations under the License.
+
+We undertake not to change the open source license (MIT license) applicable
+
+to the current version of the project delivered to anyone in the future.
+"""
+from typing import Dict
+
+from bkflow.pipeline_validate.handler import ValidatorHandler
+
+
+class ValidatorResult:
+    def __init__(self, is_valid: bool, error: str = None):
+        self.is_valid = is_valid
+        self.error = error
+
+
+class BasePipelineValidator:
+    name = None
+    validate_type = None
+
+    def __init_subclass__(cls, *args, **kwargs):
+        super().__init_subclass__(*args, **kwargs)
+
+        # 检查继承的类中是否有 validate 方法
+        if not hasattr(cls, "validate"):
+            raise ValueError(f"[{cls.__name__}] Missing required method: validate")
+
+        necessary_attrs = ["name", "validate_type"]
+        for attr in necessary_attrs:
+            if not hasattr(cls, attr) or getattr(cls, attr) is None:
+                raise ValueError(f"[{cls.__name__}] Missing required attribute: {attr}")
+
+        ValidatorHandler.register(cls)
+
+    @classmethod
+    def validate(cls, web_pipeline_tree: Dict) -> ValidatorResult:
+        raise NotImplementedError("子类必须实现 validate 方法")
+
+
+def _get_constant_display_name(const: dict, key: str) -> str:
+    """获取变量的显示名称，优先使用 name 字段"""
+    name = const.get("name", "")
+    if name:
+        return f"「{name}」({key})"
+    return f"「{key}」"

bkflow/pipeline_validate/validators/general.py

@@ -0,0 +1,85 @@
+"""
+TencentBlueKing is pleased to support the open source community by making
+蓝鲸流程引擎服务 (BlueKing Flow Engine Service) available.
+Copyright (C) 2024 THL A29 Limited,
+a Tencent company. All rights reserved.
+Licensed under the MIT License (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at http://opensource.org/licenses/MIT
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on
+an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
+either express or implied. See the License for the
+specific language governing permissions and limitations under the License.
+
+We undertake not to change the open source license (MIT license) applicable
+
+to the current version of the project delivered to anyone in the future.
+"""
+from mako.codegen import RESERVED_NAMES
+from pipeline.exceptions import PipelineException
+from pipeline.validators import validate_pipeline_tree
+
+from bkflow.constants import ValidateType
+from bkflow.pipeline_validate.validators.base import (
+    BasePipelineValidator,
+    ValidatorResult,
+)
+from bkflow.pipeline_web.parser.schemas import KEY_PATTERN_RE
+from bkflow.utils.pipeline import validate_pipeline_tree_constants
+
+
+class PipelineTreeValidator(BasePipelineValidator):
+    name = "pipeline_tree_validator"
+    validate_type = ValidateType.GENERAL.value
+
+    @classmethod
+    def validate(cls, web_pipeline_tree: dict) -> ValidatorResult:
+        try:
+            validate_pipeline_tree(web_pipeline_tree, cycle_tolerate=True)
+            return ValidatorResult(is_valid=True)
+        except Exception as e:
+            error_message = f"流程树校验失败: {str(e)}"
+            return ValidatorResult(is_valid=False, error=error_message)
+
+
+class ConstantsValidator(BasePipelineValidator):
+    """变量引用校验器，校验 pipeline tree 中 constants 的引用是否合法"""
+
+    name = "constants_validator"
+    validate_type = ValidateType.GENERAL.value
+
+    @classmethod
+    def validate(cls, web_pipeline_tree: dict) -> ValidatorResult:
+        try:
+            validate_pipeline_tree_constants(web_pipeline_tree["constants"])
+            return ValidatorResult(is_valid=True)
+        except PipelineException as e:
+            error_message = f"变量引用错误: {str(e)}"
+            return ValidatorResult(is_valid=False, error=error_message)
+
+
+class MakoKeywordValidator(BasePipelineValidator):
+    name = "mako_keyword_validator"
+    validate_type = ValidateType.TASK.value
+
+    @classmethod
+    def validate(cls, web_pipeline_tree: dict) -> ValidatorResult:
+        validation_errors = []
+
+        # 遍历所有常量变量
+        for key, const in web_pipeline_tree["constants"].items():
+            # key 格式为 ${variable_name}，需提取内部变量名再与 Mako 保留关键字比对
+            match = KEY_PATTERN_RE.match(key)
+            if not match:
+                continue
+            # 提取 ${ 和 } 之间的变量名
+            var_name = key[2:-1]
+            if var_name in RESERVED_NAMES:
+                validation_errors.append(key)
+
+        if validation_errors:
+            error_message = "变量命名校验失败: 变量 {} 使用了Mako模板引擎的保留关键字".format("; ".join(validation_errors))
+            return ValidatorResult(is_valid=False, error=error_message)
+
+        return ValidatorResult(is_valid=True)