wiz.issue: Add sourceRules field to the issue data stream (#16692)

wiz.issue: Add sourceRules field to the issue data stream

Add newly added "sourceRules" node to GraphQL query 
in issue data stream. This is mapped to "source_rules" 
field of type "nested". Mark existing "source_rule" 
field to be deprecated, but populate it with first 
sourceRule in the "source_rules" list.
This allows users time to migrate any custom artifacts 
to use "source_rules" field instead.

Implementation is largely taken from here[1].

[1] #16627 (comment)

Author: kcreddy

packages/wiz/_dev/deploy/docker/files/config-issue.yml

@@ -1,4 +1,12 @@
 # newer versions go on top
+- version: "3.12.0"
+  changes:
+    - description: |
+        Add `sourceRules` field to the issue data stream.
+        Deprecate `sourceRule` field, which will be removed in a future version.
+        Any custom-user artifacts will need to be updated to use the new field.
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/16692
 - version: "3.11.0"
   changes:
     - description: Add `risks` field to the `sourceRule` object in the issue data stream.

packages/wiz/changelog.yml

@@ -31,12 +31,12 @@ state:
          nodes {
            id
            type
-           sourceRule{
+           sourceRules{
              __typename
              ... on Control {
                id
                name
-               controlDescription: description
+               description
                resolutionRecommendation
                risks
                securitySubCategories {
@@ -52,17 +52,35 @@ state:
              ... on CloudEventRule{
                id
                name
-               cloudEventRuleDescription: description
+               description
                risks
+               securitySubCategories {
+                 title
+                 category {
+                   name
+                   framework {
+                     name
+                   }
+                 }
+               }
                sourceType
                type
              }
              ... on CloudConfigurationRule{
                id
                name
-               cloudConfigurationRuleDescription: description
+               description
                remediationInstructions
                risks
+               securitySubCategories {
+                 title
+                 category {
+                   name
+                   framework {
+                     name
+                   }
+                 }
+               }
                serviceType
              }
            }

packages/wiz/data_stream/issue/_dev/test/pipeline/test-issue.log

@@ -325,58 +325,92 @@ processors:
       tag: rename_severity
       target_field: wiz.issue.severity
       ignore_missing: true
-  - rename:
-      field: json.sourceRule.__typename
-      tag: rename_source_rule_typename
-      target_field: wiz.issue.source_rule.__typename
-      ignore_missing: true
-  - rename:
-      field: json.sourceRule.id
-      tag: rename_source_rule_id
-      target_field: wiz.issue.source_rule.id
-      ignore_missing: true
-  - rename:
-      field: json.sourceRule.controlDescription
-      tag: rename_control_description
-      target_field: wiz.issue.source_rule.control_description
-      ignore_missing: true
-  - set:
-      field: message
-      tag: set_message
-      copy_from: wiz.issue.source_rule.control_description
-      ignore_empty_value: true
-      if: ctx.message == null
-  - rename:
-      field: json.sourceRule.cloudConfigurationRuleDescription
-      tag: rename_source_rule_cloud_configuration_rule_description
-      target_field: wiz.issue.source_rule.cloud_configuration_rule_description
-      ignore_missing: true
-  - set:
-      field: message
-      tag: set_message
-      copy_from: wiz.issue.source_rule.cloud_configuration_rule_description
-      ignore_empty_value: true
-      if: ctx.message == null
-  - rename:
-      field: json.sourceRule.name
-      tag: rename_source_rule_name
-      target_field: wiz.issue.source_rule.name
-      ignore_missing: true
-  - rename:
-      field: json.sourceRule.risks
-      tag: rename_source_rule_risks
-      target_field: wiz.issue.source_rule.risks
-      ignore_missing: true
-  - rename:
-      field: json.sourceRule.resolutionRecommendation
-      tag: rename_source_rule_resolution_recommendation
-      target_field: wiz.issue.source_rule.resolution_recommendation
-      ignore_missing: true
-  - rename:
-      field: json.sourceRule.securitySubCategories
-      tag: rename_framework_name
-      target_field: wiz.issue.source_rule.security_sub_categories
-      ignore_missing: true
+  - script:
+      tag: map_source_rules_array
+      description: 'Loop over sourceRules array and create mapped objects for source_rules nested field'
+      if: ctx.json?.sourceRules != null && ctx.json.sourceRules instanceof List && ctx.json.sourceRules.size() > 0
+      lang: painless
+      source: |
+        def sourceRulesList = new ArrayList();
+        ctx.wiz.issue.source_rule = ctx.wiz.issue.source_rule ?: [:];
+        for (def rule : ctx.json.sourceRules) {
+          boolean doSourceRule = sourceRulesList.size() == 0;
+          def mappedRule = new HashMap();
+          if (rule.__typename != null) {
+            mappedRule.put('__typename', rule.__typename);
+            if (doSourceRule) {
+              ctx.wiz.issue.source_rule.__typename = rule.__typename;
+            }
+          }
+          if (rule.id != null) {
+            mappedRule.put('id', rule.id);
+            if (doSourceRule) {
+              ctx.wiz.issue.source_rule.id = rule.id;
+            }
+          }
+          if (rule.name != null) {
+            mappedRule.put('name', rule.name);
+            if (doSourceRule) {
+              ctx.wiz.issue.source_rule.name = rule.name;
+            }
+          }
+          if (rule.description != null) {
+            mappedRule.put('description', rule.description);
+            if (doSourceRule && (rule.__typename != null)) {
+              if (rule.__typename == "Control") {
+                ctx.wiz.issue.source_rule.control_description = rule.description;
+              }
+              if (rule.__typename == "CloudConfigurationEvent") {
+                ctx.wiz.issue.source_rule.control_cloud_configuration_rule_description = rule.description;
+              }
+              if (rule.__typename == "CloudEventRule") {
+                ctx.wiz.issue.source_rule.cloud_event_rule_description = rule.description;
+              }
+              if (ctx.message == null) {
+                ctx.message = rule.description;
+              }
+            }
+          }
+          if (rule.resolutionRecommendation != null) {
+            mappedRule.put('resolution_recommendation', rule.resolutionRecommendation);
+            if (doSourceRule) {
+              ctx.wiz.issue.source_rule.resolution_recommendation = rule.resolutionRecommendation;
+             }
+          }
+          if (rule.remediationInstructions != null) {
+            mappedRule.put('remediation_instructions', rule.remediationInstructions);
+          }
+          if (rule.risks != null && rule.risks.size() > 0) {
+            def risksList = new ArrayList();
+            risksList.addAll(rule.risks);
+            mappedRule.put('risks', risksList);
+            if (doSourceRule) {
+              ctx.wiz.issue.source_rule.risks = risksList;
+            }
+          }
+          if (rule.securitySubCategories != null) {
+            mappedRule.put('security_sub_categories', rule.securitySubCategories);
+            if (doSourceRule) {
+              ctx.wiz.issue.source_rule.security_sub_categories = rule.securitySubCategories;
+             }
+          }
+          if (rule.type != null) {
+            mappedRule.put('type', rule.type);
+          }
+          if (rule.serviceType != null) {
+            mappedRule.put('service_type', rule.serviceType);
+          }
+          if (rule.sourceType != null) {
+            mappedRule.put('source_type', rule.sourceType);
+          }
+          sourceRulesList.add(mappedRule);
+        }
+
+        ctx.wiz.issue.source_rules = sourceRulesList;
+      on_failure:
+        - append:
+            field: error.message
+            value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}'
   - date:
       field: json.statusChangedAt
       tag: date_set_timestamp

packages/wiz/data_stream/issue/_dev/test/pipeline/test-issue.log-expected.json

@@ -98,19 +98,71 @@
       type: keyword
     - name: source_rule
       type: group
+      description: DEPRECATED. Use `wiz.issue.source_rules` instead.
       fields:
         - name: __typename
           type: keyword
+          description: DEPRECATED. Use `wiz.issue.source_rules.__typename` instead.
         - name: control_description
           type: keyword
+          description: DEPRECATED. Use `wiz.issue.source_rules.description` instead.
         - name: cloud_configuration_rule_description
           type: keyword
+          description: DEPRECATED. Use `wiz.issue.source_rules.description` instead.
+        - name: cloud_event_rule_description
+          type: keyword
+          description: DEPRECATED. Use `wiz.issue.source_rules.description` instead.
         - name: id
           type: keyword
+          description: DEPRECATED. Use `wiz.issue.source_rules.id` instead.
         - name: name
           type: keyword
+          description: DEPRECATED. Use `wiz.issue.source_rules.name` instead.
         - name: resolution_recommendation
           type: keyword
+          description: DEPRECATED. Use `wiz.issue.source_rules.resolution_recommendation` instead.
+        - name: risks
+          type: keyword
+          description: DEPRECATED. Use `wiz.issue.source_rules.risks` instead.
+        - name: security_sub_categories
+          type: group
+          fields:
+            - name: category
+              type: group
+              fields:
+                - name: framework
+                  type: group
+                  fields:
+                    - name: name
+                      type: keyword
+                      description: DEPRECATED. Use `wiz.issue.source_rules.security_sub_categories.category.framework.name` instead.
+                - name: name
+                  type: keyword
+                  description: DEPRECATED. Use `wiz.issue.source_rules.security_sub_categories.category.name` instead.
+            - name: title
+              type: keyword
+              description: DEPRECATED. Use `wiz.issue.source_rules.security_sub_categories.title` instead.
+    - name: source_rules
+      type: nested
+      fields:
+        - name: __typename
+          type: keyword
+        - name: description
+          type: keyword
+        - name: type
+          type: keyword
+        - name: source_type
+          type: keyword
+        - name: service_type
+          type: keyword
+        - name: id
+          type: keyword
+        - name: name
+          type: keyword
+        - name: resolution_recommendation
+          type: keyword
+        - name: remediation_instructions
+          type: keyword
         - name: risks
           type: keyword
         - name: security_sub_categories