CLI support for SmartSwitch PMON

config/chassis_modules.py

@@ -30,8 +30,10 @@ def shutdown_chassis_module(db, chassis_module_name):
 
     if not chassis_module_name.startswith("SUPERVISOR") and \
        not chassis_module_name.startswith("LINE-CARD") and \
-       not chassis_module_name.startswith("FABRIC-CARD"):
-        ctx.fail("'module_name' has to begin with 'SUPERVISOR', 'LINE-CARD' or 'FABRIC-CARD'")
+       not chassis_module_name.startswith("FABRIC-CARD") and \
+       not chassis_module_name.startswith("DPU") and \
+       not chassis_module_name.startswith("SWITCH"):
+        ctx.fail("'module_name' has to begin with 'SUPERVISOR', 'LINE-CARD', 'FABRIC-CARD', 'DPU' or 'SWITCH'")
 
     fvs = {'admin_status': 'down'}
     config_db.set_entry('CHASSIS_MODULE', chassis_module_name, fvs)

show/chassis_modules.py

@@ -37,14 +37,14 @@ def status(db, chassis_module_name):
     header = ['Name', 'Description', 'Physical-Slot', 'Oper-Status', 'Admin-Status', 'Serial']
     chassis_cfg_table = db.cfgdb.get_table('CHASSIS_MODULE')
 
-    state_db = SonicV2Connector(host="127.0.0.1")
+    state_db  = SonicV2Connector(host="127.0.0.1", port="6379")
     state_db.connect(state_db.STATE_DB)
 
-    key_pattern = '*'
+    key_pattern = 'CHASSIS_MODULE_TABLE|*'
     if chassis_module_name:
-        key_pattern = '|' + chassis_module_name
+        key_pattern = 'CHASSIS_MODULE_TABLE|' + chassis_module_name
 
-    keys = state_db.keys(state_db.STATE_DB, CHASSIS_MODULE_INFO_TABLE + key_pattern)
+    keys = state_db.keys(state_db.STATE_DB, key_pattern)
     if not keys:
         print('Key {} not found in {} table'.format(key_pattern, CHASSIS_MODULE_INFO_TABLE))
         return

show/reboot_cause.py

@@ -1,6 +1,7 @@
 import json
 import os
 import sys
+import redis
 
 import click
 from tabulate import tabulate
@@ -9,7 +10,8 @@
 
 
 PREVIOUS_REBOOT_CAUSE_FILE_PATH = "/host/reboot-cause/previous-reboot-cause.json"
-
+STATE_DB=6
+CHASSIS_STATE_DB=13
 
 def read_reboot_cause_file():
     reboot_cause_dict = {}
@@ -23,6 +25,95 @@ def read_reboot_cause_file():
 
     return reboot_cause_dict
 
+# Function to fetch reboot cause data from database
+def fetch_data_from_db(module_name, fetch_history=False, use_chassis_db=False):
+    prefix='REBOOT_CAUSE|'
+    if use_chassis_db:
+        try:
+            rdb = redis.Redis(host = 'redis_chassis.server', port = 6380, decode_responses=True, db=CHASSIS_STATE_DB)
+            table_keys = rdb.keys(prefix+'*')
+        except Exception as e:
+            return []
+    else:
+        rdb = SonicV2Connector(host='127.0.0.1')
+        rdb.connect(rdb.STATE_DB, False)   # Make one attempt only
+        table_keys = rdb.keys(rdb.STATE_DB, prefix+'*')
+
+    if not table_keys is None:
+        table_keys.sort(reverse=True)
+
+    table = []
+    d = []
+    for tk in table_keys:
+        r = []
+        append = False
+        if use_chassis_db:
+            entry = rdb.hgetall(tk)
+        else:
+            entry = rdb.get_all(rdb.STATE_DB, tk)
+
+        if not module_name is None:
+            if 'device' in entry:
+                if module_name != entry['device'] and module_name != "all":
+                    continue
+                if entry['device'] in d and history == False:
+                    append = False
+                    continue
+                elif not entry['device'] in d or entry['device'] in d and history == True:
+                    append = True
+                    if not entry['device'] in d:
+                        d.append(entry['device'])
+            r.append(entry['device'] if 'device' in entry else "SWITCH")
+        suffix=""
+        if append and "DPU" in entry['device']:
+            suffix='|' + entry['device']
+        r.append(tk.replace(prefix, "").replace(suffix, ""))
+        r.append(entry['cause'] if 'cause' in entry else "")
+        r.append(entry['time'] if 'time' in entry else "")
+        r.append(entry['user'] if 'user' in entry else "")
+        if append == True and fetch_history == False:
+            table.append(r)
+        elif fetch_history == True:
+            r.append(entry['comment'] if 'comment' in entry else "")
+            if module_name is None or module_name == 'all' or module_name.startswith('SWITCH') or 'device' in entry and module_name == entry['device']:
+                table.append(r)
+
+    return table
+
+
+# Wrapper-function to fetch reboot cause data from database
+def fetch_reboot_cause_from_db(module_name):
+    table = []
+    r = []
+
+    # Read the previous reboot cause
+    reboot_cause_dict = read_reboot_cause_file()
+    reboot_gen_time = reboot_cause_dict.get("gen_time", "N/A")
+    reboot_cause = reboot_cause_dict.get("cause", "Unknown")
+    reboot_time = reboot_cause_dict.get("time", "N/A")
+    reboot_user = reboot_cause_dict.get("user", "N/A")
+
+    r.append("SWITCH")
+    r.append(reboot_gen_time if reboot_gen_time else "")
+    r.append(reboot_cause if reboot_cause else "")
+    r.append(reboot_time if reboot_time else "")
+    r.append(reboot_user if reboot_user else "")
+    table.append(r)
+
+    table += fetch_data_from_db(module_name, fetch_history=False, use_chassis_db=True)
+    return table
+
+# Function to fetch reboot cause history data from database
+def fetch_reboot_cause_history_from_db(module_name):
+    if module_name == "all":
+        # Combine data from both Redis containers for "all" modules
+        data_switch = fetch_data_from_db(module_name, fetch_history=True, use_chassis_db=False)
+        data_dpu = fetch_data_from_db(module_name, fetch_history=True, use_chassis_db=True)
+        return data_switch + data_dpu
+    elif module_name is None or module_name == "SWITCH":
+        return fetch_data_from_db(module_name, fetch_history=True, use_chassis_db=False)
+    else:
+        return fetch_data_from_db(module_name, fetch_history=True, use_chassis_db=True)
 
 #
 # 'reboot-cause' group ("show reboot-cause")
@@ -61,34 +152,24 @@ def reboot_cause(ctx):
 
         click.echo(reboot_cause_str)
 
+# 'all' command within 'reboot-cause'
+@reboot_cause.command()
+def all():
+    """Show cause of most recent reboot"""
+    reboot_cause_data = fetch_reboot_cause_from_db("all")
+    header = ['Device', 'Name', 'Cause', 'Time', 'User']
+    click.echo(tabulate(reboot_cause_data, header, numalign="left"))
 
-# 'history' subcommand ("show reboot-cause history")
+# 'history' command within 'reboot-cause'
 @reboot_cause.command()
-def history():
+@click.argument('module_name', required=False)
+def history(module_name):
     """Show history of reboot-cause"""
-    REBOOT_CAUSE_TABLE_NAME = "REBOOT_CAUSE"
-    TABLE_NAME_SEPARATOR = '|'
-    db = SonicV2Connector(host='127.0.0.1')
-    db.connect(db.STATE_DB, False)   # Make one attempt only
-    prefix = REBOOT_CAUSE_TABLE_NAME + TABLE_NAME_SEPARATOR
-    _hash = '{}{}'.format(prefix, '*')
-    table_keys = db.keys(db.STATE_DB, _hash)
-    if table_keys is not None:
-        table_keys.sort(reverse=True)
-
-        table = []
-        for tk in table_keys:
-            entry = db.get_all(db.STATE_DB, tk)
-            r = []
-            r.append(tk.replace(prefix, ""))
-            r.append(entry['cause'] if 'cause' in entry else "")
-            r.append(entry['time'] if 'time' in entry else "")
-            r.append(entry['user'] if 'user' in entry else "")
-            r.append(entry['comment'] if 'comment' in entry else "")
-            table.append(r)
-
-        header = ['Name', 'Cause', 'Time', 'User', 'Comment']
-        click.echo(tabulate(table, header, numalign="left"))
+    reboot_cause_history = fetch_reboot_cause_history_from_db(module_name)
+    if not module_name is None :
+        header = ['Device', 'Name', 'Cause', 'Time', 'User', 'Comment']
+        click.echo(tabulate(reboot_cause_history, header, numalign="left"))
     else:
-        click.echo("Reboot-cause history is not yet available in StateDB")
-        sys.exit(1)
+        header = ['Name', 'Cause', 'Time', 'User', 'Comment']
+        click.echo(tabulate(reboot_cause_history, header, numalign="left"))
+

show/system_health.py

@@ -1,10 +1,17 @@
 import os
 import sys
+import json
 
 import click
 from tabulate import tabulate
 import utilities_common.cli as clicommon
+from swsscommon.swsscommon import SonicV2Connector
+from natsort import natsorted
 
+DPU_STATE = 'DPU_STATE'
+CHASSIS_SERVER='redis_chassis.server'
+CHASSIS_SERVER_PORT=6380
+CHASSIS_STATE_DB=13
 
 def get_system_health_status():
     if os.environ.get("UTILITIES_UNIT_TESTING") == "1":
@@ -33,6 +40,120 @@ def get_system_health_status():
 
     return manager, chassis, stat
 
+def get_module_health_from_db(module_name):
+    try:
+        chassis_state_db = SonicV2Connector(host=CHASSIS_SERVER, port=CHASSIS_SERVER_PORT)
+        chassis_state_db.connect(chassis_state_db.CHASSIS_STATE_DB)
+        key = 'SYSTEM_HEALTH_INFO|'
+        suffix = '*' if not module_name or module_name == 'all' else module_name
+        key = key + suffix
+        keys = chassis_state_db.keys(chassis_state_db.CHASSIS_STATE_DB, key)
+        if not keys:
+            return
+
+        for dbkey in natsorted(keys):
+            key_list = dbkey.split('|')
+            if len(key_list) != 2:  # error data in DB, log it and ignore
+                continue
+            healthlist = []
+            modulelist = []
+            health = chassis_state_db.get_all(chassis_state_db.CHASSIS_STATE_DB, dbkey)
+            healthlist.append(health)
+            modulelist.append(key_list[1])
+        return healthlist, modulelist
+    except Exception as e:
+        click.echo("Error retrieving module health list:", e)
+    exit(1)
+
+def display_module_health_summary(module_name):
+    healthlist, modulelist = get_module_health_from_db(module_name)
+    index=0
+    for health in healthlist:
+        print("\n" + modulelist[index])
+        display_system_health_summary(json.loads(health['stat']), health['system_status_LED'])
+        index += 1
+
+def display_module_health_monitor_list(module_name):
+    healthlist, modulelist = get_module_health_from_db(module_name)
+    index=0
+    for health in healthlist:
+        print("\n" + modulelist[index])
+        display_monitor_list(json.loads(health['stat']))
+        index += 1
+
+def display_module_health_detail(module_name):
+    healthlist, modulelist = get_module_health_from_db(module_name)
+    index=0
+    for health in healthlist:
+        print("\n" + modulelist[index])
+        display_system_health_summary(json.loads(health['stat']), health['system_status_LED'])
+        display_monitor_list(json.loads(health['stat']))
+        index += 1
+
+def show_module_state(module_name):
+    chassis_state_db = SonicV2Connector(host=CHASSIS_SERVER, port=CHASSIS_SERVER_PORT)
+    chassis_state_db.connect(chassis_state_db.CHASSIS_STATE_DB)
+    key = 'DPU_STATE|'
+    suffix = '*' if not module_name or module_name == 'all' else module_name
+    key = key + suffix
+    keys = chassis_state_db.keys(chassis_state_db.CHASSIS_STATE_DB, key)
+    if not keys:
+        return
+
+    table = []
+    for dbkey in natsorted(keys):
+        key_list = dbkey.split('|')
+        if len(key_list) != 2:  # error data in DB, log it and ignore
+            continue
+        state_info = chassis_state_db.get_all(chassis_state_db.CHASSIS_STATE_DB, dbkey)
+        # Determine operational status
+        dpu_states = [value for key, value in state_info.items() if key.endswith('_state')]
+
+        midplanedown = False
+        up_cnt = 0
+        for key, value in state_info.items():
+            if  key.endswith('_state'):
+                if value.lower() == 'up':
+                    up_cnt = up_cnt + 1
+                if 'midplane' in key and value.lower() == 'down':
+                    midplanedown = True
+
+        if midplanedown:
+            oper_status = "Offline"
+        elif up_cnt == 3:
+            oper_status = "Online"
+        else:
+            oper_status = "Partial Online"
+
+        for dpustates in range(3):
+            if dpustates == 0:
+                row = [key_list[1], state_info.get('id', ''), oper_status, "", "", "", ""]
+            else:
+                row = [ "", "", "", "", "", "", ""]
+            for key, value in state_info.items():
+                if dpustates == 0 and 'midplane' in key:
+                    populate_row(row, key, value, table)
+                elif dpustates == 1 and 'control' in key:
+                    populate_row(row, key, value, table)
+                elif dpustates == 2 and 'data' in key:
+                    populate_row(row, key, value, table)
+
+    headers = ["Name", "ID", "Oper-Status", "State-Detail", "State-Value", "Time", "Reason"]
+    click.echo(tabulate(table, headers=headers))
+
+def populate_row(row, key, value, table):
+        if key.endswith('_state'):
+            row[3] = key
+            row[4] = value
+            if "up" in row[4]:
+                row[6] = ""
+            table.append(row)
+        elif key.endswith('_time'):
+            row[5] = value
+        elif key.endswith('_reason'):
+            if not "up" in row[4]:
+                row[6] = value
+
 def display_system_health_summary(stat, led):
     click.echo("System status summary\n\n  System status LED  " + led)
     services_list = []
@@ -108,27 +229,48 @@ def system_health():
     return
 
 @system_health.command()
-def summary():
+@click.argument('module_name', required=False)
+def summary(module_name):
     """Show system-health summary information"""
-    _, chassis, stat = get_system_health_status()
-    display_system_health_summary(stat, chassis.get_status_led())
-
+    if not module_name or module_name == "all":
+        if module_name == "all":
+            print("SWITCH")
+        _, chassis, stat = get_system_health_status()
+        display_system_health_summary(stat, chassis.get_status_led())
+    if module_name and module_name.startswith("DPU") or module_name == "all":
+        display_module_health_summary(module_name)
 
 @system_health.command()
-def detail():
+@click.argument('module_name', required=False)
+def detail(module_name):
     """Show system-health detail information"""
     manager, chassis, stat = get_system_health_status()
-    display_system_health_summary(stat, chassis.get_status_led())
-    display_monitor_list(stat)
-    display_ignore_list(manager)
-
+    if not module_name or module_name == "all":
+        if module_name == "all":
+            print("SWITCH")
+        display_system_health_summary(stat, chassis.get_status_led())
+        display_monitor_list(stat)
+        display_ignore_list(manager)
+    if module_name and module_name.startswith("DPU") or module_name == "all":
+        display_module_health_detail(module_name)
 
 @system_health.command()
-def monitor_list():
+@click.argument('module_name', required=False)
+def monitor_list(module_name):
     """Show system-health monitored services and devices name list"""
     _, _, stat = get_system_health_status()
-    display_monitor_list(stat)
+    if not module_name or module_name == "all":
+        if module_name == "all":
+            print("SWITCH")
+        display_monitor_list(stat)
+    if module_name and module_name.startswith("DPU") or module_name == "all":
+        display_module_health_monitor_list(module_name)
 
+@system_health.command()
+@click.argument('module_name', required=False)
+def dpu(module_name):
+    """Show system-health dpu information"""
+    show_module_state(module_name)
 
 @system_health.group('sysready-status',invoke_without_command=True)
 @click.pass_context