diff --git a/aci-preupgrade-validation-script.py b/aci-preupgrade-validation-script.py index ebe0477..4481bd6 100644 --- a/aci-preupgrade-validation-script.py +++ b/aci-preupgrade-validation-script.py @@ -6007,6 +6007,57 @@ def apic_vmm_inventory_sync_faults_check(**kwargs): recommended_action=recommended_action, doc_url=doc_url) + +@check_wrapper(check_title='AAA Provider DNS Name Configuration check') +def aaa_snmpd_dns_provider_check(tversion, **kwargs): + result = PASS + headers = ["Provider Type", "Provider Name", "Provider DN"] + data = [] + recommended_action = 'Contact Cisco TAC for Support before upgrade' + doc_url = 'https://datacenter.github.io/ACI-Pre-Upgrade-Validation-Script/validations/#aaa-provider-dns-name-configuration-check' + + if not tversion: + return Result(result=MANUAL, msg=TVER_MISSING) + + if tversion.older_than("6.1(5e)"): + + # Query for all AAA providers (TACACS+, RADIUS, and LDAP) + provider_api = 'uni.json?query-target=subtree&target-subtree-class=aaaTacacsPlusProvider,aaaRadiusProvider,aaaLdapProvider' + providers = icurl('mo', provider_api) + + # Regular expression to detect DNS names (not IP addresses) + dns_name_pattern = r'[a-zA-Z]' + + for provider in providers: + provider_type = None + provider_name = None + provider_dn = None + + if 'aaaRadiusProvider' in provider: + provider_type = 'RADIUS' + provider_name = provider['aaaRadiusProvider']['attributes']['name'] + provider_dn = provider['aaaRadiusProvider']['attributes']['dn'] + elif 'aaaLdapProvider' in provider: + provider_type = 'LDAP' + provider_name = provider['aaaLdapProvider']['attributes']['name'] + provider_dn = provider['aaaLdapProvider']['attributes']['dn'] + elif 'aaaTacacsPlusProvider' in provider: + provider_type = 'TACACS+' + provider_name = provider['aaaTacacsPlusProvider']['attributes']['name'] + provider_dn = provider['aaaTacacsPlusProvider']['attributes']['dn'] + + # Check if the provider name contains DNS name (has alphabetic characters) + if provider_name and re.search(dns_name_pattern, provider_name): + data.append([provider_type, provider_name, provider_dn]) + + if data: + result = FAIL_O + else: + return Result(result=PASS, msg=VER_NOT_AFFECTED) + + return Result(result=result, headers=headers, data=data, recommended_action=recommended_action, doc_url=doc_url) + + # ---- Script Execution ---- @@ -6168,6 +6219,7 @@ class CheckManager: standby_sup_sync_check, isis_database_byte_check, configpush_shard_check, + aaa_snmpd_dns_provider_check, ] ssh_checks = [ diff --git a/docs/docs/validations.md b/docs/docs/validations.md index fa1fc0e..dbf1843 100644 --- a/docs/docs/validations.md +++ b/docs/docs/validations.md @@ -191,6 +191,7 @@ Items | Defect | This Script [Stale pconsRA Object][d26] | CSCwp22212 | :warning:{title="Deprecated"} | :no_entry_sign: [ISIS DTEPs Byte Size][d27] | CSCwp15375 | :white_check_mark: | :no_entry_sign: [Policydist configpushShardCont Crash][d28] | CSCwp95515 | :white_check_mark: | +[AAA Provider DNS Name Configuration check][d29] | CSCwq57598 | :white_check_mark: | :no_entry_sign: [d1]: #ep-announce-compatibility [d2]: #eventmgr-db-size-defect-susceptibility @@ -220,6 +221,7 @@ Items | Defect | This Script [d26]: #stale-pconsra-object [d27]: #isis-dteps-byte-size [d28]: #policydist-configpushshardcont-crash +[d29]: #aaa-provider-dns-name-configuration-check ## General Check Details @@ -2614,6 +2616,14 @@ Due to [CSCwp95515][59], upgrading to an affected version while having any `conf If any instances of `configpushShardCont` are flagged by this script, Cisco TAC must be contacted to identify and resolve the underlying issue before performing the upgrade. +### AAA Provider DNS Name Configuration check + +Due to `CSCwq57598`, spines hit kernel panic after the upgrade. SNMPd main process acquires a lock for memory operations at the same time, child SNMP process is spawned by the DNS Cache to resolve the AAA server name & inherently acquires parent mutex_lock state & eventually ends up in blocked state. + +This script identifies if APIC is configured with AAA Authentication providers (TACACS+, RADIUS, and LDAP) using hostname. If detected, contact cisco TAC support and upgrade to the fix version. + + + [0]: https://github.com/datacenter/ACI-Pre-Upgrade-Validation-Script [1]: https://www.cisco.com/c/dam/en/us/td/docs/Website/datacenter/apicmatrix/index.html [2]: https://www.cisco.com/c/en/us/support/switches/nexus-9000-series-switches/products-release-notes-list.html diff --git a/tests/checks/aaa_snmpd_dns_provider_check/README.md b/tests/checks/aaa_snmpd_dns_provider_check/README.md new file mode 100644 index 0000000..d42063e --- /dev/null +++ b/tests/checks/aaa_snmpd_dns_provider_check/README.md @@ -0,0 +1,118 @@ +# Test Cases for aaa_snmpd_dns_provider_check + +## Overview +This directory contains test cases for the `aaa_snmpd_dns_provider_check` function, which validates AAA provider configurations to prevent snmpd memory exhaustion issues (CSCwq57598). + +## Test Function +**Function Name:** `aaa_snmpd_dns_provider_check` +**Bug Reference:** CSCwq57598 - hundreds of snmpd process exhaust memory and lead kernel panic with oom + +## Test Scenarios + +### Test Case 1: No Target Version Provided +- **Input:** No target version (None) +- **JSON File:** `providers_with_dns.json` +- **Expected Result:** `MANUAL CHECK REQUIRED` +- **Expected Data Count:** 0 +- **Description:** When target version is not provided, the check should return MANUAL and indicate that target version is missing. + +### Test Case 2: Target Version 6.1(5e) - Not Affected +- **Input:** Target version `6.1(5e)` +- **JSON File:** `providers_with_dns.json` +- **Expected Result:** `PASS` +- **Expected Data Count:** 0 +- **Description:** Version 6.1(5e) includes the fix, so the check should pass with "Version not affected" message. + +### Test Case 3: Target Version Newer Than 6.1(5e) - Not Affected +- **Input:** Target version `6.2(1a)` +- **JSON File:** `providers_with_dns.json` +- **Expected Result:** `PASS` +- **Expected Data Count:** 0 +- **Description:** Versions newer than 6.1(5e) are not affected, so the check should pass. + +### Test Case 4: Target Version Older Than 6.1(5e) with DNS Names - FAIL +- **Input:** Target version `6.1(4a)` +- **JSON File:** `providers_with_dns.json` +- **Expected Result:** `FAIL_O` (FAIL - OUTAGE WARNING!!) +- **Expected Data Count:** 2 +- **Description:** When target version is older than 6.1(5e) and DNS names are found in AAA providers, the check should fail. +- **Flagged Providers:** + - RADIUS provider: `rad.cisco.com` + - LDAP provider: `aaa.domain.com` + +### Test Case 5: Target Version Older Than 6.1(5e) with Only IP Addresses - PASS +- **Input:** Target version `6.1(3a)` +- **JSON File:** `providers_with_ips.json` +- **Expected Result:** `PASS` +- **Expected Data Count:** 0 +- **Description:** When only IP addresses are configured, the check should pass even on affected versions. +- **Providers (all IPs):** + - TACACS+ provider: `10.0.0.1` + - RADIUS provider: `192.168.1.100` + - LDAP provider: `10.10.10.50` + +### Test Case 6: Target Version Older Than 6.1(5e) with Mixed (IP and DNS) - FAIL +- **Input:** Target version `6.0(5a)` +- **JSON File:** `providers_mixed.json` +- **Expected Result:** `FAIL_O` (FAIL - OUTAGE WARNING!!) +- **Expected Data Count:** 2 +- **Description:** When a mix of IP and DNS names is configured, only DNS entries should be flagged. +- **Flagged Providers:** + - RADIUS provider: `radius.example.com` + - LDAP provider: `ldap-server.local` +- **Not Flagged:** + - TACACS+ provider: `10.0.0.1` (IP address) + +### Test Case 7: Target Version Older Than 6.1(5e) with No Providers - PASS +- **Input:** Target version `5.2(8a)` +- **JSON File:** `providers_empty.json` +- **Expected Result:** `PASS` +- **Expected Data Count:** 0 +- **Description:** When no AAA providers are configured, the check should pass. + +## JSON Test Data Files + +### providers_with_dns.json +Contains AAA providers with DNS names: +- RADIUS provider: `rad.cisco.com` +- LDAP provider: `aaa.domain.com` + +### providers_with_ips.json +Contains AAA providers with only IP addresses: +- TACACS+ provider: `10.0.0.1` +- RADIUS provider: `192.168.1.100` +- LDAP provider: `10.10.10.50` + +### providers_mixed.json +Contains a mix of IP and DNS configurations: +- TACACS+ provider: `10.0.0.1` (IP) +- RADIUS provider: `radius.example.com` (DNS) +- LDAP provider: `ldap-server.local` (DNS) + +### providers_empty.json +Empty array - no AAA providers configured. + +## Running the Tests + +```bash +# Run all tests +cd /data/ssd/dhaselva/repo/ACI-Escalation/ACI-Pre-Upgrade-Validation-Script +python3 -m pytest tests/checks/aaa_snmpd_dns_provider_check/test_aaa_snmpd_dns_provider_check.py -v + +# Run with detailed output +python3 -m pytest tests/checks/aaa_snmpd_dns_provider_check/test_aaa_snmpd_dns_provider_check.py -v -s +``` + +## Test Results +All 7 test cases pass successfully, validating: +1. Version checking logic (missing version, affected versions, not affected versions) +2. DNS name detection (alphabetic characters in provider names) +3. Proper handling of IP addresses (should not be flagged) +4. Mixed configurations (only DNS names should be flagged) +5. Empty configurations (no providers) + +## Expected Output Format +When DNS names are detected in affected versions: +- **Headers:** ["Provider Type", "Provider Name", "Provider DN"] +- **Data:** List of providers with DNS names +- **Recommended Action:** "Replace DNS names with IP addresses in AAA provider configurations to prevent snmpd memory exhaustion" diff --git a/tests/checks/aaa_snmpd_dns_provider_check/providers_empty.json b/tests/checks/aaa_snmpd_dns_provider_check/providers_empty.json new file mode 100644 index 0000000..fe51488 --- /dev/null +++ b/tests/checks/aaa_snmpd_dns_provider_check/providers_empty.json @@ -0,0 +1 @@ +[] diff --git a/tests/checks/aaa_snmpd_dns_provider_check/providers_mixed.json b/tests/checks/aaa_snmpd_dns_provider_check/providers_mixed.json new file mode 100644 index 0000000..625ab00 --- /dev/null +++ b/tests/checks/aaa_snmpd_dns_provider_check/providers_mixed.json @@ -0,0 +1,106 @@ +[ + { + "aaaTacacsPlusProvider": { + "attributes": { + "SSLValidationLevel": "permissive", + "annotation": "", + "authProtocol": "pap", + "childAction": "", + "descr": "", + "dn": "uni/userext/tacacsext/tacacsplusprovider-10.0.0.1", + "enableTLS": "no", + "epgDn": "", + "extMngdBy": "", + "keyring": "", + "lcOwn": "local", + "modTs": "2025-12-17T05:26:53.215+00:00", + "monPolDn": "uni/fabric/monfab-default", + "monitorServer": "disabled", + "monitoringUser": "default", + "name": "10.0.0.1", + "nameAlias": "", + "operState": "unknown", + "ownerKey": "", + "ownerTag": "", + "port": "49", + "retries": "1", + "snmpIndex": "1", + "status": "", + "timeout": "5", + "tp": "", + "uid": "15374", + "userdom": ":all:", + "vrfName": "" + } + } + }, + { + "aaaRadiusProvider": { + "attributes": { + "annotation": "", + "authPort": "1812", + "authProtocol": "pap", + "childAction": "", + "descr": "", + "dn": "uni/userext/radiusext/radiusprovider-radius.example.com", + "epgDn": "", + "extMngdBy": "", + "includeMsgAuth": "no", + "lcOwn": "local", + "modTs": "2025-12-23T05:40:49.902+00:00", + "monPolDn": "uni/fabric/monfab-default", + "monitorServer": "disabled", + "monitoringUser": "default", + "name": "radius.example.com", + "nameAlias": "", + "operState": "unknown", + "ownerKey": "", + "ownerTag": "", + "retries": "1", + "snmpIndex": "3", + "status": "", + "timeout": "5", + "uid": "15374", + "userdom": ":all:", + "vrfName": "" + } + } + }, + { + "aaaLdapProvider": { + "attributes": { + "SSLValidationLevel": "strict", + "annotation": "", + "attribute": "CiscoAVPair", + "authMethod": "LdapBind", + "basedn": "", + "childAction": "", + "descr": "test", + "dn": "uni/userext/ldapext/ldapprovider-ldap-server.local", + "enableSSL": "no", + "epgDn": "", + "extMngdBy": "", + "filter": "cn=$userid", + "lcOwn": "local", + "modTs": "2025-12-23T05:39:47.423+00:00", + "monPolDn": "uni/fabric/monfab-default", + "monitorServer": "disabled", + "monitoringUser": "default", + "name": "ldap-server.local", + "nameAlias": "", + "operState": "unknown", + "ownerKey": "", + "ownerTag": "", + "port": "389", + "retries": "1", + "rootdn": "", + "snmpIndex": "2", + "status": "", + "timeout": "30", + "uid": "15374", + "userdom": ":all:", + "vrfName": "" + } + } + } +] diff --git a/tests/checks/aaa_snmpd_dns_provider_check/providers_with_dns.json b/tests/checks/aaa_snmpd_dns_provider_check/providers_with_dns.json new file mode 100644 index 0000000..74ddc9d --- /dev/null +++ b/tests/checks/aaa_snmpd_dns_provider_check/providers_with_dns.json @@ -0,0 +1,71 @@ +[ + { + "aaaRadiusProvider": { + "attributes": { + "annotation": "", + "authPort": "1812", + "authProtocol": "pap", + "childAction": "", + "descr": "", + "dn": "uni/userext/radiusext/radiusprovider-rad.cisco.com", + "epgDn": "", + "extMngdBy": "", + "includeMsgAuth": "no", + "lcOwn": "local", + "modTs": "2025-12-23T05:40:49.902+00:00", + "monPolDn": "uni/fabric/monfab-default", + "monitorServer": "disabled", + "monitoringUser": "default", + "name": "rad.cisco.com", + "nameAlias": "", + "operState": "unknown", + "ownerKey": "", + "ownerTag": "", + "retries": "1", + "snmpIndex": "3", + "status": "", + "timeout": "5", + "uid": "15374", + "userdom": ":all:", + "vrfName": "" + } + } + }, + { + "aaaLdapProvider": { + "attributes": { + "SSLValidationLevel": "strict", + "annotation": "", + "attribute": "CiscoAVPair", + "authMethod": "LdapBind", + "basedn": "", + "childAction": "", + "descr": "test", + "dn": "uni/userext/ldapext/ldapprovider-aaa.domain.com", + "enableSSL": "no", + "epgDn": "", + "extMngdBy": "", + "filter": "cn=$userid", + "lcOwn": "local", + "modTs": "2025-12-23T05:39:47.423+00:00", + "monPolDn": "uni/fabric/monfab-default", + "monitorServer": "disabled", + "monitoringUser": "default", + "name": "aaa.domain.com", + "nameAlias": "", + "operState": "unknown", + "ownerKey": "", + "ownerTag": "", + "port": "389", + "retries": "1", + "rootdn": "", + "snmpIndex": "2", + "status": "", + "timeout": "30", + "uid": "15374", + "userdom": ":all:", + "vrfName": "" + } + } + } +] diff --git a/tests/checks/aaa_snmpd_dns_provider_check/providers_with_ips.json b/tests/checks/aaa_snmpd_dns_provider_check/providers_with_ips.json new file mode 100644 index 0000000..bf3b215 --- /dev/null +++ b/tests/checks/aaa_snmpd_dns_provider_check/providers_with_ips.json @@ -0,0 +1,106 @@ +[ + { + "aaaTacacsPlusProvider": { + "attributes": { + "SSLValidationLevel": "permissive", + "annotation": "", + "authProtocol": "pap", + "childAction": "", + "descr": "", + "dn": "uni/userext/tacacsext/tacacsplusprovider-10.0.0.1", + "enableTLS": "no", + "epgDn": "", + "extMngdBy": "", + "keyring": "", + "lcOwn": "local", + "modTs": "2025-12-17T05:26:53.215+00:00", + "monPolDn": "uni/fabric/monfab-default", + "monitorServer": "disabled", + "monitoringUser": "default", + "name": "10.0.0.1", + "nameAlias": "", + "operState": "unknown", + "ownerKey": "", + "ownerTag": "", + "port": "49", + "retries": "1", + "snmpIndex": "1", + "status": "", + "timeout": "5", + "tp": "", + "uid": "15374", + "userdom": ":all:", + "vrfName": "" + } + } + }, + { + "aaaRadiusProvider": { + "attributes": { + "annotation": "", + "authPort": "1812", + "authProtocol": "pap", + "childAction": "", + "descr": "", + "dn": "uni/userext/radiusext/radiusprovider-192.168.1.100", + "epgDn": "", + "extMngdBy": "", + "includeMsgAuth": "no", + "lcOwn": "local", + "modTs": "2025-12-23T05:40:49.902+00:00", + "monPolDn": "uni/fabric/monfab-default", + "monitorServer": "disabled", + "monitoringUser": "default", + "name": "192.168.1.100", + "nameAlias": "", + "operState": "unknown", + "ownerKey": "", + "ownerTag": "", + "retries": "1", + "snmpIndex": "3", + "status": "", + "timeout": "5", + "uid": "15374", + "userdom": ":all:", + "vrfName": "" + } + } + }, + { + "aaaLdapProvider": { + "attributes": { + "SSLValidationLevel": "strict", + "annotation": "", + "attribute": "CiscoAVPair", + "authMethod": "LdapBind", + "basedn": "", + "childAction": "", + "descr": "test", + "dn": "uni/userext/ldapext/ldapprovider-10.10.10.50", + "enableSSL": "no", + "epgDn": "", + "extMngdBy": "", + "filter": "cn=$userid", + "lcOwn": "local", + "modTs": "2025-12-23T05:39:47.423+00:00", + "monPolDn": "uni/fabric/monfab-default", + "monitorServer": "disabled", + "monitoringUser": "default", + "name": "10.10.10.50", + "nameAlias": "", + "operState": "unknown", + "ownerKey": "", + "ownerTag": "", + "port": "389", + "retries": "1", + "rootdn": "", + "snmpIndex": "2", + "status": "", + "timeout": "30", + "uid": "15374", + "userdom": ":all:", + "vrfName": "" + } + } + } +] diff --git a/tests/checks/aaa_snmpd_dns_provider_check/test_aaa_snmpd_dns_provider_check.py b/tests/checks/aaa_snmpd_dns_provider_check/test_aaa_snmpd_dns_provider_check.py new file mode 100644 index 0000000..ac87a80 --- /dev/null +++ b/tests/checks/aaa_snmpd_dns_provider_check/test_aaa_snmpd_dns_provider_check.py @@ -0,0 +1,74 @@ +import os +import pytest +import logging +import importlib +from helpers.utils import read_data + +script = importlib.import_module("aci-preupgrade-validation-script") + +log = logging.getLogger(__name__) +dir = os.path.dirname(os.path.abspath(__file__)) + +test_function = "aaa_snmpd_dns_provider_check" + +# icurl queries +providers_api = 'uni.json?query-target=subtree&target-subtree-class=aaaTacacsPlusProvider,aaaRadiusProvider,aaaLdapProvider' + + +@pytest.mark.parametrize( + "icurl_outputs, tversion, expected_result, expected_data_count", + [ + # Test case 1: No target version provided - should return MANUAL + ( + {providers_api: read_data(dir, "providers_with_dns.json")}, + None, + script.MANUAL, + 0, + ), + # Test case 2: Target version 6.1(5e) or newer - should return PASS (not affected) + ( + {providers_api: read_data(dir, "providers_with_dns.json")}, + "6.1(5e)", + script.PASS, + 0, + ), + # Test case 3: Target version newer than 6.1(5e) - should return PASS (not affected) + ( + {providers_api: read_data(dir, "providers_with_dns.json")}, + "6.2(1a)", + script.PASS, + 0, + ), + # Test case 4: Target version older than 6.1(5e) with DNS names - should return FAIL_O + ( + {providers_api: read_data(dir, "providers_with_dns.json")}, + "6.1(4a)", + script.FAIL_O, + 2, + ), + # Test case 5: Target version older than 6.1(5e) with only IP addresses - should return PASS + ( + {providers_api: read_data(dir, "providers_with_ips.json")}, + "6.1(3a)", + script.PASS, + 0, + ), + # Test case 6: Target version older than 6.1(5e) with mixed (IP and DNS) - should return FAIL_O + ( + {providers_api: read_data(dir, "providers_mixed.json")}, + "6.0(5a)", + script.FAIL_O, + 2, + ), + # Test case 7: Target version older than 6.1(5e) with no providers - should return PASS + ( + {providers_api: read_data(dir, "providers_empty.json")}, + "5.2(8a)", + script.PASS, + 0, + ), + ], +) +def test_logic(run_check, mock_icurl, tversion, expected_result, expected_data_count): + result = run_check(tversion=script.AciVersion(tversion) if tversion else None) + assert result.result == expected_result