diff --git a/aci-preupgrade-validation-script.py b/aci-preupgrade-validation-script.py index ebe0477..bc29fbf 100644 --- a/aci-preupgrade-validation-script.py +++ b/aci-preupgrade-validation-script.py @@ -151,6 +151,7 @@ def __init__(self, hostname): self._term_len = 0 # terminal length for cisco devices self._login = False # set to true at first successful login self._log = None # private variable for tracking logfile state + self.bind_ip = None # optional source IP to bind for SSH def __connected(self): # determine if a connection is already open @@ -207,6 +208,8 @@ def connect(self): "spawning new pexpect connection: ssh %s@%s -p %d" % (self.username, self.hostname, self.port)) no_verify = " -o StrictHostKeyChecking=no -o LogLevel=ERROR -o UserKnownHostsFile=/dev/null" if self.verify: no_verify = "" + if self.bind_ip: + no_verify += " -b %s" % self.bind_ip self.child = pexpect.spawn("ssh %s %s@%s -p %d" % (no_verify, self.username, self.hostname, self.port), searchwindowsize=self.searchwindowsize) elif self.protocol.lower() == "telnet": @@ -5970,7 +5973,6 @@ def configpush_shard_check(tversion, **kwargs): return Result(result=result, headers=headers, data=data, recommended_action=recommended_action, doc_url=doc_url) - @check_wrapper(check_title='APIC VMM inventory sync fault (F0132)') def apic_vmm_inventory_sync_faults_check(**kwargs): result = PASS @@ -6007,6 +6009,108 @@ def apic_vmm_inventory_sync_faults_check(**kwargs): recommended_action=recommended_action, doc_url=doc_url) +@check_wrapper(check_title="HW Changes bit check for specific node model") +def HW_changes_bit_check(tversion, username, password, fabric_nodes, **kwargs): + result = PASS + headers = ["Node", "Model", "HW Changes Bits", "Recommended Action"] + data = [] + recommended_action = "Contact Cisco TAC for Support before upgrade" + doc_url = "https://datacenter.github.io/ACI-Pre-Upgrade-Validation-Script/validations/#HW_changes_bit_check" + + if not tversion: + return Result(result=MANUAL, msg=TVER_MISSING) + if not tversion.newer_than("14.2(4a)"): + return Result(result=NA, msg=VER_NOT_AFFECTED) + + affected_models = {"N9K-C9316D-GX", "N9K-C93600CD-GX"} + + node_found = any( + node["fabricNode"]["attributes"]["model"] in affected_models + for node in fabric_nodes + ) + + if not node_found: + return Result(result=PASS, msg="No switch models found") + + # Discover APIC IP (bind source) + try: + apic_hostname = run_cmd("bash -c \"hostname\"", splitlines=True)[0].strip() + if not apic_hostname: + return Result(result=ERROR, msg="Could not determine APIC hostname") + + apic_ip = next( + (node["fabricNode"]["attributes"].get("address") + for node in fabric_nodes + if node["fabricNode"]["attributes"]["name"] == apic_hostname), + None + ) + except Exception as e: + return Result(result=ERROR, msg="Failed to get APIC IP: {}".format(e)) + + if not apic_ip: + return Result(result=ERROR, msg="Could not determine APIC IP from fabricNode attributes") + + hw_bits_re = re.compile(r"HW Changes Bits\s*:\s*(0x[0-9a-fA-F]+)") + has_error = False + + # SSH directly to each switch hostname from APIC, binding source to APIC IP (-b ) + for node in fabric_nodes: + if node["fabricNode"]["attributes"]["model"] not in ["N9K-C9316D-GX", "N9K-C93600CD-GX"]: + continue + attr = node['fabricNode']['attributes'] + node_name = attr['name'] + node_model = attr['model'] + + node_title = "Checking {} ({})...".format(node_name, node_model) + try: + c = Connection(node_name) + c.username = username + c.password = password + c.log = LOG_FILE + c.bind_ip = apic_ip # enables: ssh -b + c.connect() + except Exception as e: + data.append([node_name, node_model, "-", "Connection Error: {}".format(str(e))]) + has_error = True + continue + + try: + # Execute command to check HW Changes Bits + c.cmd("vsh -c 'show sprom cpu-info' | grep \"HW Changes Bits\"") + raw = c.output.strip() + match = hw_bits_re.search(raw) + if not match: + data.append([node_name, node_model, "Parse Error", "Unable to parse HW Changes Bits"]) + has_error = True + else: + hw_bits = match.group(1) + val = int(hw_bits, 16) + if val < 2: + result = FAIL_O + data.append([node_name, node_model, hw_bits, "This switch need Manual Upgrade (CSCvv04251). Contact TAC for Support"]) + except Exception as e: + data.append([ + node_name, + node_model, + '-', + 'Failed to check HW Changes Bits: {}. Please check MANUALLY.'.format(str(e)) + ]) + has_error = True + continue + finally: + try: + c.close() + except Exception: + pass + + if has_error and result == PASS: + result = ERROR + elif not data: + result = PASS + + return Result(result=result, headers=headers, data=data, recommended_action=recommended_action, doc_url=doc_url) + + # ---- Script Execution ---- @@ -6168,6 +6272,7 @@ class CheckManager: standby_sup_sync_check, isis_database_byte_check, configpush_shard_check, + HW_changes_bit_check, ] ssh_checks = [ diff --git a/docs/docs/validations.md b/docs/docs/validations.md index fa1fc0e..b36b9f2 100644 --- a/docs/docs/validations.md +++ b/docs/docs/validations.md @@ -191,6 +191,7 @@ Items | Defect | This Script [Stale pconsRA Object][d26] | CSCwp22212 | :warning:{title="Deprecated"} | :no_entry_sign: [ISIS DTEPs Byte Size][d27] | CSCwp15375 | :white_check_mark: | :no_entry_sign: [Policydist configpushShardCont Crash][d28] | CSCwp95515 | :white_check_mark: | +[HW_changes_bit_check] | CSCvv04251 | :white_check_mark: | :no_entry_sign: [d1]: #ep-announce-compatibility [d2]: #eventmgr-db-size-defect-susceptibility @@ -220,6 +221,7 @@ Items | Defect | This Script [d26]: #stale-pconsra-object [d27]: #isis-dteps-byte-size [d28]: #policydist-configpushshardcont-crash +[d29]: #HW_changes_bit_check ## General Check Details @@ -2614,6 +2616,15 @@ Due to [CSCwp95515][59], upgrading to an affected version while having any `conf If any instances of `configpushShardCont` are flagged by this script, Cisco TAC must be contacted to identify and resolve the underlying issue before performing the upgrade. +### HW changes bit check + +The HW Changes Bit Check is a validation step in the ACI Pre-Upgrade Validation script that verifies the status of `HW Changes Bits` indicators within the ACI fabric before an upgrade. + +If discrepancies are detected, the function raises alerts to prompt administrators to investigate and resolve these issues prior to proceeding with the upgrade. This proactive measure helps prevent potential failures or complications during the upgrade process, ensuring a smoother transition to the new software version. + +For more detailed information, refer to the defect report [CSCvv04251][62], which outlines specific scenarios and resolutions related to hardware change indicators. + + [0]: https://github.com/datacenter/ACI-Pre-Upgrade-Validation-Script [1]: https://www.cisco.com/c/dam/en/us/td/docs/Website/datacenter/apicmatrix/index.html [2]: https://www.cisco.com/c/en/us/support/switches/nexus-9000-series-switches/products-release-notes-list.html @@ -2676,3 +2687,4 @@ If any instances of `configpushShardCont` are flagged by this script, Cisco TAC [59]: https://bst.cloudapps.cisco.com/bugsearch/bug/CSCwp95515 [60]: https://www.cisco.com/c/en/us/solutions/collateral/data-center-virtualization/application-centric-infrastructure/white-paper-c11-743951.html#Inter [61]: https://www.cisco.com/c/en/us/solutions/collateral/data-center-virtualization/application-centric-infrastructure/white-paper-c11-743951.html#EnablePolicyCompression +[62]: https://bst.cloudapps.cisco.com/bugsearch/bug/CSCvv04251 diff --git a/tests/checks/HW_changes_bit_check/FabricNodes_matching.json b/tests/checks/HW_changes_bit_check/FabricNodes_matching.json new file mode 100644 index 0000000..ef4d3db --- /dev/null +++ b/tests/checks/HW_changes_bit_check/FabricNodes_matching.json @@ -0,0 +1,290 @@ +[ + { + "fabricNode": { + "attributes": { + "adSt": "on", + "address": "12.3.136.65", + "annotation": "", + "apicType": "not-applicable", + "childAction": "", + "delayedHeartbeat": "no", + "dn": "topology/pod-3/node-303", + "extMngdBy": "", + "fabricSt": "active", + "id": "303", + "lastStateModTs": "2025-12-11T21:35:56.699-08:00", + "lcOwn": "local", + "mfgTm": "2019-01-13T16:00:00.000-08:00", + "modTs": "2025-12-11T21:35:56.754-08:00", + "model": "N9K-C93180YC-FX", + "monPolDn": "uni/fabric/monfab-default", + "name": "ifav42-leaf15", + "nameAlias": "", + "nodeType": "unspecified", + "role": "leaf", + "serial": "FDO2303039T", + "status": "", + "uid": "0", + "userdom": "all", + "vendor": "Cisco Systems, Inc", + "version": "n9000-16.2(0.167)" + } + } + }, + { + "fabricNode": { + "attributes": { + "adSt": "on", + "address": "11.3.208.65", + "annotation": "", + "apicType": "not-applicable", + "childAction": "", + "delayedHeartbeat": "no", + "dn": "topology/pod-2/node-302", + "extMngdBy": "", + "fabricSt": "active", + "id": "302", + "lastStateModTs": "2025-12-11T18:34:54.381-08:00", + "lcOwn": "local", + "mfgTm": "2019-01-27T16:00:00.000-08:00", + "modTs": "2025-12-11T18:34:54.436-08:00", + "model": "N9K-C9336C-FX2", + "monPolDn": "uni/fabric/monfab-default", + "name": "ifav42-leaf12", + "nameAlias": "", + "nodeType": "unspecified", + "role": "leaf", + "serial": "FDO23051B9A", + "status": "", + "uid": "0", + "userdom": "all", + "vendor": "Cisco Systems, Inc", + "version": "n9000-16.2(0.167)" + } + } + }, + { + "fabricNode": { + "attributes": { + "adSt": "on", + "address": "10.0.224.65", + "annotation": "", + "apicType": "not-applicable", + "childAction": "", + "delayedHeartbeat": "no", + "dn": "topology/pod-1/node-3001", + "extMngdBy": "", + "fabricSt": "active", + "id": "3001", + "lastStateModTs": "2025-12-10T10:29:29.923-08:00", + "lcOwn": "local", + "mfgTm": "2023-03-12T16:00:00.000-08:00", + "modTs": "2025-12-10T10:29:29.978-08:00", + "model": "N9K-C9316D-GX", + "monPolDn": "uni/fabric/monfab-default", + "name": "ifav42-spine6", + "nameAlias": "", + "nodeType": "unspecified", + "role": "spine", + "serial": "FDO27111UE3", + "status": "", + "uid": "0", + "userdom": "all", + "vendor": "Cisco Systems, Inc", + "version": "n9000-16.2(0.167)" + } + } + }, + { + "fabricNode": { + "attributes": { + "adSt": "on", + "address": "11.3.208.64", + "annotation": "", + "apicType": "not-applicable", + "childAction": "", + "delayedHeartbeat": "no", + "dn": "topology/pod-2/node-3002", + "extMngdBy": "", + "fabricSt": "active", + "id": "3002", + "lastStateModTs": "2025-12-10T10:29:29.733-08:00", + "lcOwn": "local", + "mfgTm": "2019-03-10T16:00:00.000-08:00", + "modTs": "2025-12-10T10:29:29.787-08:00", + "model": "N9K-C93600CD-GX", + "monPolDn": "uni/fabric/monfab-default", + "name": "ifav42-spine8", + "nameAlias": "", + "nodeType": "unspecified", + "role": "spine", + "serial": "FDO2311073Q", + "status": "", + "uid": "0", + "userdom": "all", + "vendor": "Cisco Systems, Inc", + "version": "n9000-16.1(4h)" + } + } + }, + { + "fabricNode": { + "attributes": { + "adSt": "on", + "address": "10.0.224.64", + "annotation": "", + "apicType": "not-applicable", + "childAction": "", + "delayedHeartbeat": "no", + "dn": "topology/pod-1/node-301", + "extMngdBy": "", + "fabricSt": "active", + "id": "301", + "lastStateModTs": "2025-12-10T10:29:29.194-08:00", + "lcOwn": "local", + "mfgTm": "2019-01-06T16:00:00.000-08:00", + "modTs": "2025-12-10T10:29:29.246-08:00", + "model": "N9K-C93180YC-FX", + "monPolDn": "uni/fabric/monfab-default", + "name": "ifav42-leaf11", + "nameAlias": "", + "nodeType": "unspecified", + "role": "leaf", + "serial": "FDO23020NL0", + "status": "", + "uid": "0", + "userdom": "all", + "vendor": "Cisco Systems, Inc", + "version": "n9000-16.2(0.167)" + } + } + }, + { + "fabricNode": { + "attributes": { + "adSt": "on", + "address": "12.3.136.64", + "annotation": "", + "apicType": "not-applicable", + "childAction": "", + "delayedHeartbeat": "no", + "dn": "topology/pod-3/node-3003", + "extMngdBy": "", + "fabricSt": "active", + "id": "3003", + "lastStateModTs": "2025-12-10T10:29:29.194-08:00", + "lcOwn": "local", + "mfgTm": "2019-03-03T16:00:00.000-08:00", + "modTs": "2025-12-10T10:29:29.246-08:00", + "model": "N9K-C93600CD-GX", + "monPolDn": "uni/fabric/monfab-default", + "name": "ifav42-spine9", + "nameAlias": "", + "nodeType": "unspecified", + "role": "spine", + "serial": "FDO23100QX8", + "status": "", + "uid": "0", + "userdom": "all", + "vendor": "Cisco Systems, Inc", + "version": "n9000-16.2(0.167)" + } + } + }, + { + "fabricNode": { + "attributes": { + "adSt": "on", + "address": "10.0.0.3", + "annotation": "", + "apicType": "apic", + "childAction": "", + "delayedHeartbeat": "no", + "dn": "topology/pod-3/node-3", + "extMngdBy": "", + "fabricSt": "commissioned", + "id": "3", + "lastStateModTs": "2025-12-09T23:49:02.749-08:00", + "lcOwn": "local", + "mfgTm": "1969-12-31T16:00:00.000-08:00", + "modTs": "2025-12-09T23:49:03.039-08:00", + "model": "APIC-SERVER-M4T", + "monPolDn": "uni/fabric/monfab-default", + "name": "ifav42-ifc7", + "nameAlias": "", + "nodeType": "unspecified", + "role": "controller", + "serial": "WZP27160Q34", + "status": "", + "uid": "0", + "userdom": "all", + "vendor": "Cisco Systems, Inc", + "version": "6.2(0.198)" + } + } + }, + { + "fabricNode": { + "attributes": { + "adSt": "on", + "address": "10.0.0.2", + "annotation": "", + "apicType": "apic", + "childAction": "", + "delayedHeartbeat": "no", + "dn": "topology/pod-2/node-2", + "extMngdBy": "", + "fabricSt": "commissioned", + "id": "2", + "lastStateModTs": "2025-12-09T23:45:18.617-08:00", + "lcOwn": "local", + "mfgTm": "1969-12-31T16:00:00.000-08:00", + "modTs": "2025-12-09T23:45:59.424-08:00", + "model": "APIC-SERVER-M4T", + "monPolDn": "uni/fabric/monfab-default", + "name": "ifav42-ifc6", + "nameAlias": "", + "nodeType": "unspecified", + "role": "controller", + "serial": "WZP27150MYQ", + "status": "", + "uid": "0", + "userdom": "all", + "vendor": "Cisco Systems, Inc", + "version": "6.2(0.198)" + } + } + }, + { + "fabricNode": { + "attributes": { + "adSt": "on", + "address": "10.0.0.1", + "annotation": "", + "apicType": "apic", + "childAction": "", + "delayedHeartbeat": "no", + "dn": "topology/pod-1/node-1", + "extMngdBy": "", + "fabricSt": "commissioned", + "id": "1", + "lastStateModTs": "2025-12-09T23:37:37.436-08:00", + "lcOwn": "local", + "mfgTm": "1969-12-31T16:00:00.000-08:00", + "modTs": "2025-12-09T23:38:59.204-08:00", + "model": "APIC-SERVER-M2", + "monPolDn": "uni/fabric/monfab-default", + "name": "ifav42-ifc5", + "nameAlias": "", + "nodeType": "unspecified", + "role": "controller", + "serial": "FCH2313V0KV", + "status": "", + "uid": "0", + "userdom": "all", + "vendor": "Cisco Systems, Inc", + "version": "6.2(0.198)" + } + } + } +] \ No newline at end of file diff --git a/tests/checks/HW_changes_bit_check/test_HW_changes_bit_check.py b/tests/checks/HW_changes_bit_check/test_HW_changes_bit_check.py new file mode 100644 index 0000000..fca2578 --- /dev/null +++ b/tests/checks/HW_changes_bit_check/test_HW_changes_bit_check.py @@ -0,0 +1,249 @@ +import os +import pytest +import logging +import importlib +from helpers.utils import read_data + +script = importlib.import_module("aci-preupgrade-validation-script") +log = logging.getLogger(__name__) +dir = os.path.dirname(os.path.abspath(__file__)) + +test_function = "HW_changes_bit_check" + +# Define API endpoints +fabricNode_api = 'fabricNode.json' +fabricNode_api += '?query-target-filter=or(eq(fabricNode.model,"N9K-C9316D-GX"),eq(fabricNode.model,"N9K-C93600CD-GX"))' + + +fabricNode = read_data(dir, "FabricNodes_matching.json") + +node_names = [ + mo["fabricNode"]["attributes"]["name"] + for mo in fabricNode +] + +hostname_cmd = "bash -c \"hostname\"" + +hostname_output = "ifav42-ifc5" + +hw_changes_bit_cmd = "vsh -c 'show sprom cpu-info' | grep \"HW Changes Bits\"" + +hw_changes_lower_bit_output = """\ +vsh -c 'show sprom cpu-info' | grep 'HW Changes Bits' + HW Changes Bits : 0x0 +leaf12#""" + +hw_changes_higher_bit_output = """\ +vsh -c 'show sprom cpu-info' | grep 'HW Changes Bits' + HW Changes Bits : 0x3 +leaf12#""" + +hw_changes_no_output = """\ +vsh -c 'show sprom cpu-info' | grep 'HW Changes Bits' + +leaf12#""" + + +@pytest.mark.parametrize( + "fabric_nodes, conn_failure, conn_cmds, cmd_outputs, tversion, expected_result", + [ + # tversion not given + ( + [], + False, + {}, + {hostname_cmd: {"splitlines": True, "output": hostname_output}}, + None, + script.MANUAL, + ), + + # Connection failure + ( + read_data(dir, "FabricNodes_matching.json"), + True, + {}, + {hostname_cmd: {"splitlines": True, "output": hostname_output}}, + "14.2(5a)", + script.ERROR, + ), + + # No Matching model nodes found + ( + [], + False, + {}, + {hostname_cmd: {"splitlines": True, "output": hostname_output}}, + "14.2(5a)", + script.PASS, + ), + + # All nodes with HW Changes Bits = 0x0 + ( + read_data(dir, "FabricNodes_matching.json"), + False, + { + node_name: [ + { + "cmd": hw_changes_bit_cmd, + "output": hw_changes_lower_bit_output, + "exception": None, + } + ] + for node_name in node_names + }, + {hostname_cmd: {"splitlines": True, "output": hostname_output}}, + "14.2(5a)", + script.FAIL_O, + ), + + # All nodes with HW Changes Bits = 0x3 + ( + read_data(dir, "FabricNodes_matching.json"), + False, + { + node_name: [ + { + "cmd": hw_changes_bit_cmd, + "output": hw_changes_higher_bit_output, + "exception": None, + } + ] + for node_name in node_names + }, + {hostname_cmd: {"splitlines": True, "output": hostname_output}}, + "14.2(5a)", + script.PASS, + ), + + # One node fails (0x0), another node pass (0x3) + ( + read_data(dir, "FabricNodes_matching.json"), + False, + { + node_names[0]: [ + { + "cmd": hw_changes_bit_cmd, + "output": hw_changes_lower_bit_output, + "exception": None, + } + ], + node_names[1]: [ + { + "cmd": hw_changes_bit_cmd, + "output": hw_changes_higher_bit_output, + "exception": None, + } + ], + node_names[2]: [ + { + "cmd": hw_changes_bit_cmd, + "output": hw_changes_lower_bit_output, + "exception": None, + } + ], + node_names[3]: [ + { + "cmd": hw_changes_bit_cmd, + "output": hw_changes_higher_bit_output, + "exception": None, + } + ] + }, + {hostname_cmd: {"splitlines": True, "output": hostname_output}}, + "14.2(5a)", + script.FAIL_O, + ), + + # No output from command execution + ( + read_data(dir, "FabricNodes_matching.json"), + False, + { + node_name: [ + { + "cmd": hw_changes_bit_cmd, + "output": hw_changes_no_output, + "exception": None, + } + ] + for node_name in node_names + }, + {hostname_cmd: {"splitlines": True, "output": hostname_output}}, + "14.2(5a)", + script.ERROR, + ), + + # One node connection fails, others succeed + ( + read_data(dir, "FabricNodes_matching.json"), + False, + { + node_names[0]: [ + { + "cmd": hw_changes_bit_cmd, + "output": hw_changes_lower_bit_output, + "exception": None, + } + ], + node_names[1]: [ + { + "cmd": hw_changes_bit_cmd, + "output": "", + "exception": Exception("connection_failure"), + } + ], + node_names[2]: [ + { + "cmd": hw_changes_bit_cmd, + "output": hw_changes_lower_bit_output, + "exception": None, + } + ], + node_names[3]: [ + { + "cmd": hw_changes_bit_cmd, + "output": hw_changes_higher_bit_output, + "exception": None, + } + ] + }, + {hostname_cmd: {"splitlines": True, "output": hostname_output}}, + "14.2(5a)", + script.FAIL_O, + ), + + # Command execution exception + ( + read_data(dir, "FabricNodes_matching.json"), + False, + { + node_name: [ + { + "cmd": hw_changes_bit_cmd, + "output": "", + "exception": Exception("Command execution failed"), + } + ] + for node_name in node_names + }, + {hostname_cmd: {"splitlines": True, "output": hostname_output}}, + "14.2(5a)", + script.ERROR, + ), + + # ERROR when APIC hostname cannot be determined (empty ifconfig output) + ( + read_data(dir, "FabricNodes_matching.json"), + False, + {}, + {hostname_cmd: {"splitlines": True, "output": ""}}, + "14.2(5a)", + script.ERROR, + ), + ], +) + +def test_logic(run_check, fabric_nodes, mock_conn, mock_run_cmd, tversion, expected_result): + + result = run_check(tversion=script.AciVersion(tversion) if tversion else None, username=None, password=None, fabric_nodes=fabric_nodes) + assert result.result == expected_result \ No newline at end of file diff --git a/tests/test_get_vpc_node.py b/tests/test_get_vpc_node.py index 0f664a5..613e458 100644 --- a/tests/test_get_vpc_node.py +++ b/tests/test_get_vpc_node.py @@ -56,4 +56,4 @@ def test_get_vpc_nodes(capsys, mock_icurl, expected_result, expected_stdout): captured = capsys.readouterr() print(captured.out) - assert captured.out == expected_stdout + assert captured.out == expected_stdout \ No newline at end of file