Added pre-upgrade validation for N9K-C9408 with more than 6 N9K-X9400-16W LEM's for the bug CSCws82819

aci-preupgrade-validation-script.py

@@ -6053,6 +6053,55 @@ def auto_firmware_update_on_switch_check(cversion, tversion, **kwargs):
 
     return Result(result=result, headers=headers, data=data, recommended_action=recommended_action, doc_url=doc_url)
 
+@check_wrapper(check_title='N9K-C9408 with more than 5 N9K-X9400-16W LEMs')
+def n9k_c9408_model_lem_count_check(tversion, fabric_nodes, **kwargs):
+    result = PASS
+    headers = ["Node ID", "Switch Model", "LEM Model", "LEM Count"]
+    data = []
+    recommended_action = (
+        "N9K-C9408 switches configured with >5 N9K-X9400-16W LEMs will enter a boot loop if upgraded to impacted release of CSCws82819. Upgrade to Fix version or Use less than 6 LEMS on impacted release"
+    )
+    doc_url = 'https://datacenter.github.io/ACI-Pre-Upgrade-Validation-Script/validations/#n9k-c9408-with-more-than-5-n9k-x9400-16w-lems'
+
+    if tversion.older_than("6.1(2f)") or (tversion.newer_than("6.1(5e)") and not tversion.same_as("6.2(1g)")):
+        return Result(result=NA, msg=VER_NOT_AFFECTED)
+
+    affected_nodes = {}
+    for node in fabric_nodes:
+        node_id = node['fabricNode']['attributes']['id']
+        model = node['fabricNode']['attributes']['model']
+        if model == "N9K-C9408":
+            affected_nodes[node_id] = "N9K-C9408"
+
+    if not affected_nodes:
+        return Result(result=NA, msg='No N9K-C9408 nodes found. Skipping.')
+
+    eqptLC_api = 'eqptLC.json?query-target-filter=eq(eqptLC.model,"N9K-X9400-16W")'
+    try:
+        eqptLCs = icurl('class', eqptLC_api)
+    except Exception as e:
+        return Result(result=ERROR, msg="Failed to query {}: {}".format(eqptLC_api, e))
+
+    lem_count_per_node = defaultdict(int)
+    for eqptLC in eqptLCs:
+        dn = eqptLC['eqptLC']['attributes']['dn']
+        dn_match = re.search(node_regex, dn)
+        if not dn_match:
+            continue
+        node_id = dn_match.group("node")
+        if node_id in affected_nodes:
+            lem_count_per_node[node_id] += 1
+
+    for node_id in sorted(affected_nodes, key=int):
+        lem_count = lem_count_per_node[node_id]
+        if lem_count > 5:
+            data.append([node_id, affected_nodes[node_id], "N9K-X9400-16W", lem_count])
+
+    if data:
+        result = FAIL_O
+
+    return Result(result=result, headers=headers, data=data, recommended_action=recommended_action, doc_url=doc_url)
+
 # ---- Script Execution ----
 
 
@@ -6216,6 +6265,7 @@ class CheckManager:
         isis_database_byte_check,
         configpush_shard_check,
         auto_firmware_update_on_switch_check,
+        n9k_c9408_model_lem_count_check,
 
     ]
     ssh_checks = [
@@ -6386,3 +6436,4 @@ def main(_args=None):
         prints(msg)
         log.error(msg, exc_info=True)
         sys.exit(1)
+

docs/docs/validations.md

@@ -194,6 +194,7 @@ Items                                           | Defect       | This Script
 [ISIS DTEPs Byte Size][d27]                     | CSCwp15375   | :white_check_mark: | :no_entry_sign:
 [Policydist configpushShardCont Crash][d28]     | CSCwp95515   | :white_check_mark: | :no_entry_sign:
 [Auto Firmware Update on Switch Discovery][d29] | CSCwe83941   | :white_check_mark: | :no_entry_sign:
+[N9K-C9408 with more than 5 N9K-X9400-16W LEMs][d30]     | CSCws82819   | :white_check_mark: | :no_entry_sign:
 
 [d1]: #ep-announce-compatibility
 [d2]: #eventmgr-db-size-defect-susceptibility
@@ -224,6 +225,7 @@ Items                                           | Defect       | This Script
 [d27]: #isis-dteps-byte-size
 [d28]: #policydist-configpushshardcont-crash
 [d29]: #auto-firmware-update-on-switch-discovery
+[d30]: #n9k-c9408-with-more-than-5-n9k-x9400-16w-lems
 
 ## General Check Details
 
@@ -2667,6 +2669,11 @@ To avoid this risk, consider disabling Auto Firmware Update before upgrading to
 !!! note
     This issue occurs because older switch firmware versions are not compatible with switch images 6.0(3) or newer. The APIC version is not a factor.
 
+### N9K-C9408 with more than 5 N9K-X9400-16W LEMs
+
+Due to defect [CSCws82819][64], N9K-C9408 switch will experience a boot loop with dt_helper process crash if upgraded to versions 16.1(2f) to 16.1(5) or 16.2(1g) with more than 5 N9K-X9400-16W LEMs installed.
+
+To avoid this issue, please upgrade to fix version or use less than 6 N9K-X9400-16W in one spine node.
 
 [0]: https://github.com/datacenter/ACI-Pre-Upgrade-Validation-Script
 [1]: https://www.cisco.com/c/dam/en/us/td/docs/Website/datacenter/apicmatrix/index.html
@@ -2731,4 +2738,5 @@ To avoid this risk, consider disabling Auto Firmware Update before upgrading to
 [60]: https://www.cisco.com/c/en/us/solutions/collateral/data-center-virtualization/application-centric-infrastructure/white-paper-c11-743951.html#Inter
 [61]: https://www.cisco.com/c/en/us/solutions/collateral/data-center-virtualization/application-centric-infrastructure/white-paper-c11-743951.html#EnablePolicyCompression
 [62]: https://bst.cloudapps.cisco.com/bugsearch/bug/CSCwe83941
-[63]: https://www.cisco.com/c/en/us/td/docs/dcn/aci/apic/all/apic-installation-aci-upgrade-downgrade/Cisco-APIC-Installation-ACI-Upgrade-Downgrade-Guide/m-auto-firmware-update.html
+[63]: https://www.cisco.com/c/en/us/td/docs/dcn/aci/apic/all/apic-installation-aci-upgrade-downgrade/Cisco-APIC-Installation-ACI-Upgrade-Downgrade-Guide/m-auto-firmware-update.html
+[64]: https://bst.cloudapps.cisco.com/bugsearch/bug/CSCws82819

tests/checks/n9k_c9408_model_lem_count_check/eqptLC_5_node.json

@@ -0,0 +1,7 @@
+[
+  {"eqptLC": {"attributes": {"dn": "topology/pod-1/node-101/sys/ch/lcslot-1/lc", "model": "N9K-X9400-16W"}}},
+  {"eqptLC": {"attributes": {"dn": "topology/pod-1/node-101/sys/ch/lcslot-2/lc", "model": "N9K-X9400-16W"}}},
+  {"eqptLC": {"attributes": {"dn": "topology/pod-1/node-101/sys/ch/lcslot-3/lc", "model": "N9K-X9400-16W"}}},
+  {"eqptLC": {"attributes": {"dn": "topology/pod-1/node-101/sys/ch/lcslot-4/lc", "model": "N9K-X9400-16W"}}},
+  {"eqptLC": {"attributes": {"dn": "topology/pod-1/node-101/sys/ch/lcslot-5/lc", "model": "N9K-X9400-16W"}}}
+]

tests/checks/n9k_c9408_model_lem_count_check/eqptLC_6_node.json

@@ -0,0 +1,8 @@
+[
+  {"eqptLC": {"attributes": {"dn": "topology/pod-1/node-101/sys/ch/lcslot-1/lc", "model": "N9K-X9400-16W"}}},
+  {"eqptLC": {"attributes": {"dn": "topology/pod-1/node-101/sys/ch/lcslot-2/lc", "model": "N9K-X9400-16W"}}},
+  {"eqptLC": {"attributes": {"dn": "topology/pod-1/node-101/sys/ch/lcslot-3/lc", "model": "N9K-X9400-16W"}}},
+  {"eqptLC": {"attributes": {"dn": "topology/pod-1/node-101/sys/ch/lcslot-4/lc", "model": "N9K-X9400-16W"}}},
+  {"eqptLC": {"attributes": {"dn": "topology/pod-1/node-101/sys/ch/lcslot-5/lc", "model": "N9K-X9400-16W"}}},
+  {"eqptLC": {"attributes": {"dn": "topology/pod-1/node-101/sys/ch/lcslot-6/lc", "model": "N9K-X9400-16W"}}}
+]

tests/checks/n9k_c9408_model_lem_count_check/eqptLC_7_node.json

@@ -0,0 +1,58 @@
+[
+  {
+    "eqptLC": {
+      "attributes": {
+        "dn": "topology/pod-1/node-101/sys/ch/lcslot-1/lc",
+        "model": "N9K-X9400-16W"
+      }
+    }
+  },
+  {
+    "eqptLC": {
+      "attributes": {
+        "dn": "topology/pod-1/node-101/sys/ch/lcslot-2/lc",
+        "model": "N9K-X9400-16W"
+      }
+    }
+  },
+  {
+    "eqptLC": {
+      "attributes": {
+        "dn": "topology/pod-1/node-101/sys/ch/lcslot-3/lc",
+        "model": "N9K-X9400-16W"
+      }
+    }
+  },
+  {
+    "eqptLC": {
+      "attributes": {
+        "dn": "topology/pod-1/node-101/sys/ch/lcslot-4/lc",
+        "model": "N9K-X9400-16W"
+      }
+    }
+  },
+  {
+    "eqptLC": {
+      "attributes": {
+        "dn": "topology/pod-1/node-101/sys/ch/lcslot-5/lc",
+        "model": "N9K-X9400-16W"
+      }
+    }
+  },
+  {
+    "eqptLC": {
+      "attributes": {
+        "dn": "topology/pod-1/node-101/sys/ch/lcslot-6/lc",
+        "model": "N9K-X9400-16W"
+      }
+    }
+  },
+  {
+    "eqptLC": {
+      "attributes": {
+        "dn": "topology/pod-1/node-101/sys/ch/lcslot-7/lc",
+        "model": "N9K-X9400-16W"
+      }
+    }
+  }
+]

tests/checks/n9k_c9408_model_lem_count_check/eqptLC_empty.json

@@ -0,0 +1 @@
+[]

tests/checks/n9k_c9408_model_lem_count_check/eqptLC_mixed.json

@@ -0,0 +1,15 @@
+[
+  {"eqptLC": {"attributes": {"dn": "topology/pod-1/node-101/sys/ch/lcslot-1/lc", "model": "N9K-X9400-16W"}}},
+  {"eqptLC": {"attributes": {"dn": "topology/pod-1/node-101/sys/ch/lcslot-2/lc", "model": "N9K-X9400-16W"}}},
+  {"eqptLC": {"attributes": {"dn": "topology/pod-1/node-101/sys/ch/lcslot-3/lc", "model": "N9K-X9400-16W"}}},
+  {"eqptLC": {"attributes": {"dn": "topology/pod-1/node-101/sys/ch/lcslot-4/lc", "model": "N9K-X9400-16W"}}},
+  {"eqptLC": {"attributes": {"dn": "topology/pod-1/node-101/sys/ch/lcslot-5/lc", "model": "N9K-X9400-16W"}}},
+  {"eqptLC": {"attributes": {"dn": "topology/pod-1/node-101/sys/ch/lcslot-6/lc", "model": "N9K-X9400-16W"}}},
+  {"eqptLC": {"attributes": {"dn": "topology/pod-1/node-102/sys/ch/lcslot-1/lc", "model": "N9K-X9400-16W"}}},
+  {"eqptLC": {"attributes": {"dn": "topology/pod-1/node-102/sys/ch/lcslot-2/lc", "model": "N9K-X9400-16W"}}},
+  {"eqptLC": {"attributes": {"dn": "topology/pod-1/node-102/sys/ch/lcslot-3/lc", "model": "N9K-X9400-16W"}}},
+  {"eqptLC": {"attributes": {"dn": "topology/pod-1/node-102/sys/ch/lcslot-4/lc", "model": "N9K-X9400-16W"}}},
+  {"eqptLC": {"attributes": {"dn": "topology/pod-1/node-102/sys/ch/lcslot-5/lc", "model": "N9K-X9400-16W"}}},
+  {"eqptLC": {"attributes": {"dn": "topology/pod-1/node-102/sys/ch/lcslot-6/lc", "model": "N9K-X9400-16W"}}},
+  {"eqptLC": {"attributes": {"dn": "topology/pod-1/node-201/sys/ch/lcslot-1/lc", "model": "N9K-X9400-16W"}}}
+]

tests/checks/n9k_c9408_model_lem_count_check/fabricNode_mixed.json

@@ -0,0 +1,26 @@
+[
+  {
+    "fabricNode": {
+      "attributes": {
+        "id": "101",
+        "model": "N9K-C9408"
+      }
+    }
+  },
+  {
+    "fabricNode": {
+      "attributes": {
+        "id": "102",
+        "model": "N9K-C9408"
+      }
+    }
+  },
+  {
+    "fabricNode": {
+      "attributes": {
+        "id": "201",
+        "model": "N9K-C93180YC-FX"
+      }
+    }
+  }
+]

tests/checks/n9k_c9408_model_lem_count_check/fabricNode_n9k_c9408.json

@@ -0,0 +1,10 @@
+[
+  {
+    "fabricNode": {
+      "attributes": {
+        "id": "101",
+        "model": "N9K-C9408"
+      }
+    }
+  }
+]

tests/checks/n9k_c9408_model_lem_count_check/fabricNode_no_n9k_c9408.json

@@ -0,0 +1,10 @@
+[
+  {
+    "fabricNode": {
+      "attributes": {
+        "id": "101",
+        "model": "N9K-C93180YC-FX"
+      }
+    }
+  }
+]

tests/checks/n9k_c9408_model_lem_count_check/test_n9k_c9408_model_lem_count_check.py

@@ -0,0 +1,136 @@
+import os
+import pytest
+import logging
+import importlib
+from helpers.utils import read_data
+
+script = importlib.import_module("aci-preupgrade-validation-script")
+
+log = logging.getLogger(__name__)
+dir = os.path.dirname(os.path.abspath(__file__))
+
+test_function = "n9k_c9408_model_lem_count_check"
+
+# icurl queries
+eqptLC_api = 'eqptLC.json?query-target-filter=eq(eqptLC.model,"N9K-X9400-16W")'
+
+
+@pytest.mark.parametrize(
+    "icurl_outputs, tversion, fabric_nodes, expected_result, expected_data, expected_msg",
+    [
+        # Version not affected (lower than 6.1(2f))
+        (
+            {eqptLC_api: read_data(dir, "eqptLC_empty.json")},
+            "6.1(2e)",
+            read_data(dir, "fabricNode_n9k_c9408.json"),
+            script.NA,
+            [],
+            script.VER_NOT_AFFECTED,
+        ),
+        # Version not affected (higher than 6.2(1g))
+        (
+            {eqptLC_api: read_data(dir, "eqptLC_empty.json")},
+            "6.2(1h)",
+            read_data(dir, "fabricNode_n9k_c9408.json"),
+            script.NA,
+            [],
+            script.VER_NOT_AFFECTED,
+        ),
+        # Applicable version but no N9K-C9408 nodes found
+        (
+            {eqptLC_api: read_data(dir, "eqptLC_6_node.json")},
+            "6.1(2f)",
+            read_data(dir, "fabricNode_no_n9k_c9408.json"),
+            script.NA,
+            [],
+            "No N9K-C9408 nodes found. Skipping.",
+        ),
+        # Applicable version, C9408 exists, no LEM entries
+        (
+            {eqptLC_api: read_data(dir, "eqptLC_empty.json")},
+            "6.1(2f)",
+            read_data(dir, "fabricNode_n9k_c9408.json"),
+            script.PASS,
+            [],
+            "",
+        ),
+        # Applicable version, C9408 exists, with <=5 LEMs -> PASS
+        (
+            {eqptLC_api: read_data(dir, "eqptLC_5_node.json")},
+            "6.2(1g)",
+            read_data(dir, "fabricNode_n9k_c9408.json"),
+            script.PASS,
+            [],
+            "",
+        ),
+        # Applicable version with 6 LEMs -> FAIL_O
+        (
+            {eqptLC_api: read_data(dir, "eqptLC_6_node.json")},
+            "6.2(1g)",
+            read_data(dir, "fabricNode_n9k_c9408.json"),
+            script.FAIL_O,
+            [["101", "N9K-C9408", "N9K-X9400-16W", 6]],
+            "",
+        ),
+        # Applicable mid-train version 6.1(5e), less than 6 LEMs on C9408 -> PASS
+        (
+            {eqptLC_api: read_data(dir, "eqptLC_5_node.json")},
+            "6.1(5e)",
+            read_data(dir, "fabricNode_n9k_c9408.json"),
+            script.PASS,
+            [],
+            "",
+        ),
+        # Applicable mid-train version 6.1(5e), 6 LEMs on C9408 -> FAIL_O
+        (
+            {eqptLC_api: read_data(dir, "eqptLC_6_node.json")},
+            "6.1(5e)",
+            read_data(dir, "fabricNode_n9k_c9408.json"),
+            script.FAIL_O,
+            [["101", "N9K-C9408", "N9K-X9400-16W", 6]],
+            "",
+        ),
+        # Version not affected (fixed after 6.1(5e))
+        (
+            {eqptLC_api: read_data(dir, "eqptLC_6_node.json")},
+            "6.1(5f)",
+            read_data(dir, "fabricNode_n9k_c9408.json"),
+            script.NA,
+            [],
+            script.VER_NOT_AFFECTED,
+        ),
+        # Applicable version, 6 LEMs on C9408 -> FAIL_O
+        (
+            {eqptLC_api: read_data(dir, "eqptLC_6_node.json")},
+            "6.1(2f)",
+            read_data(dir, "fabricNode_n9k_c9408.json"),
+            script.FAIL_O,
+            [["101", "N9K-C9408", "N9K-X9400-16W", 6]],
+            "",
+        ),
+        # Count only C9408 nodes and only matching LEM model
+        (
+            {eqptLC_api: read_data(dir, "eqptLC_mixed.json")},
+            "6.1(3a)",
+            read_data(dir, "fabricNode_mixed.json"),
+            script.FAIL_O,
+            [
+                ["101", "N9K-C9408", "N9K-X9400-16W", 6],
+                ["102", "N9K-C9408", "N9K-X9400-16W", 6],
+            ],
+            "",
+        ),
+    ],
+)
+def test_logic(run_check, mock_icurl, icurl_outputs, tversion, fabric_nodes, expected_result, expected_data, expected_msg):
+    result = run_check(
+        tversion=script.AciVersion(tversion),
+        fabric_nodes=fabric_nodes,
+    )
+
+    assert result.result == expected_result
+    assert result.data == expected_data
+    assert result.msg == expected_msg
+
+    if expected_result == script.FAIL_O:
+        assert "boot loop" in result.recommended_action