#!/bin/bash

# 硬件健康检测脚本
# 支持：
# 1. SATA/SAS 直通盘、HBA/JBOD 模式盘 SMART 检测
# 2. NVMe 磨损/健康检测
# 3. LSI/Broadcom MegaRAID/Avago RAID 卡物理盘检测，优先使用 storcli/perccli，兼容 MegaCli
# 4. 内存 ECC/内核日志错误检测

# ======================== 可调整阈值 ========================
NVME_WEAR_WARN=${NVME_WEAR_WARN:-80}          # NVMe Percentage Used >= 80% 预警
NVME_WEAR_CRIT=${NVME_WEAR_CRIT:-100}         # NVMe Percentage Used >= 100% 严重
NVME_TEMP_WARN=${NVME_TEMP_WARN:-70}           # NVMe 温度 >= 70℃ 预警
NVME_TEMP_CRIT=${NVME_TEMP_CRIT:-80}           # NVMe 温度 >= 80℃ 严重

DISK_TEMP_WARN=${DISK_TEMP_WARN:-55}           # SATA/SAS 温度 >= 55℃ 预警
DISK_TEMP_CRIT=${DISK_TEMP_CRIT:-60}           # SATA/SAS 温度 >= 60℃ 严重

# RAID 卡后面物理盘用 smartctl -d megaraid,N 时，需要一个 RAID 虚拟盘设备作为入口。
# 默认自动取第一个 /dev/sdX。如环境特殊，可执行前指定：
# RAID_SMART_DEVICE=/dev/sda ./disk_mem_nvme_sas_raid_check.sh
RAID_SMART_DEVICE=${RAID_SMART_DEVICE:-}

TS=$(date +%s)

# 日志文件
DISK_LOG="/tmp/disk_health_${TS}.log"
NVME_LOG="/tmp/nvme_health_${TS}.log"
RAID_LOG="/tmp/raid_health_${TS}.log"
MEM_LOG="/tmp/mem_health_${TS}.log"

bad_disks=()
bad_nvmes=()
bad_raids=()

# ======================== 通用函数 ========================
num_only() {
    # 提取第一个数字，处理 1,234、80%、32 Celsius 等格式
    echo "$1" | tr -d ',%' | grep -oE '[0-9]+' | head -n 1
}

trim() {
    echo "$1" | sed 's/^[[:space:]]*//;s/[[:space:]]*$//'
}

print_line() {
    local log_file=$1
    echo "----------------------------------------" | tee -a "$log_file"
}

need_cmd() {
    if ! command -v "$1" >/dev/null 2>&1; then
        return 1
    fi
    return 0
}

find_tool() {
    local tool_name=$1
    shift

    if command -v "$tool_name" >/dev/null 2>&1; then
        command -v "$tool_name"
        return 0
    fi

    local p
    for p in "$@"; do
        if [ -x "$p" ]; then
            echo "$p"
            return 0
        fi
    done
    return 1
}

get_smart_attr_raw() {
    local output=$1
    local attr=$2
    echo "$output" | awk -v attr="$attr" '
        $2 == attr {
            for (i=10; i<=NF; i++) printf $i " ";
            print "";
            exit
        }'
}

get_smart_temp() {
    local output=$1
    local temp

    temp=$(num_only "$(echo "$output" | awk -F: '/Current Drive Temperature:/ {print $2; exit}')")
    if [ -n "$temp" ]; then
        echo "$temp"
        return 0
    fi

    temp=$(num_only "$(get_smart_attr_raw "$output" "Temperature_Celsius")")
    if [ -n "$temp" ]; then
        echo "$temp"
        return 0
    fi

    temp=$(num_only "$(get_smart_attr_raw "$output" "Airflow_Temperature_Cel")")
    if [ -n "$temp" ]; then
        echo "$temp"
        return 0
    fi

    temp=$(num_only "$(echo "$output" | awk -F: '/^Temperature:/ {print $2; exit}')")
    [ -n "$temp" ] && echo "$temp"
}

get_first_block_disk_device() {
    if [ -n "$RAID_SMART_DEVICE" ] && [ -b "$RAID_SMART_DEVICE" ]; then
        echo "$RAID_SMART_DEVICE"
        return 0
    fi

    local dev
    dev=$(lsblk -d -n -o NAME,TYPE 2>/dev/null | awk '$2=="disk" && $1 ~ /^sd[a-z]+$/ {print "/dev/"$1; exit}')
    [ -n "$dev" ] && echo "$dev"
}

# ======================== SMART 分析函数：SATA/SAS 通用 ========================
analyze_ata_sas_smart_output() {
    local label=$1
    local log_file=$2
    local output=$3
    local issue=0

    local model serial protocol rotation health temp power_hours
    model=$(echo "$output" | awk -F: '/^Device Model:|^Product:|^Model Number:/ {sub(/^[ \t]+/, "", $2); print $2; exit}')
    serial=$(echo "$output" | awk -F: '/^Serial Number:|^Serial number:/ {sub(/^[ \t]+/, "", $2); print $2; exit}')
    protocol=$(echo "$output" | awk -F: '/^Transport protocol:/ {sub(/^[ \t]+/, "", $2); print $2; exit}')
    rotation=$(echo "$output" | awk -F: '/^Rotation Rate:/ {sub(/^[ \t]+/, "", $2); print $2; exit}')
    health=$(echo "$output" | grep -iE "SMART overall-health self-assessment test result|SMART Health Status" | head -n 1)
    temp=$(get_smart_temp "$output")
    power_hours=$(num_only "$(get_smart_attr_raw "$output" "Power_On_Hours")")
    [ -z "$power_hours" ] && power_hours=$(num_only "$(echo "$output" | awk -F: '/number of hours powered up|Accumulated power on time/ {print $2; exit}')")

    echo "检测对象: $label" | tee -a "$log_file"
    echo "型号: ${model:-未知}" | tee -a "$log_file"
    echo "序列号: ${serial:-未知}" | tee -a "$log_file"
    echo "协议/接口: ${protocol:-未知}" | tee -a "$log_file"
    echo "转速/介质: ${rotation:-未知}" | tee -a "$log_file"
    echo "通电时间: ${power_hours:-未知} 小时" | tee -a "$log_file"
    echo "温度: ${temp:-未知}℃" | tee -a "$log_file"

    if [ -n "$health" ]; then
        echo "SMART 健康状态: $health" | tee -a "$log_file"
        if ! echo "$health" | grep -qiE "PASSED|OK"; then
            echo "!!! SMART 整体健康状态异常 !!!" | tee -a "$log_file"
            issue=1
        fi
    else
        echo "警告: 无法获取 SMART 整体健康状态，可能是 RAID 虚拟盘或设备不支持" | tee -a "$log_file"
    fi

    # ATA/SATA 常见关键属性
    local reallocated pending offline_uncorrect reported_uncorrect end_to_end cmd_timeout crc
    reallocated=$(num_only "$(get_smart_attr_raw "$output" "Reallocated_Sector_Ct")")
    pending=$(num_only "$(get_smart_attr_raw "$output" "Current_Pending_Sector")")
    offline_uncorrect=$(num_only "$(get_smart_attr_raw "$output" "Offline_Uncorrectable")")
    reported_uncorrect=$(num_only "$(get_smart_attr_raw "$output" "Reported_Uncorrect")")
    end_to_end=$(num_only "$(get_smart_attr_raw "$output" "End-to-End_Error")")
    cmd_timeout=$(num_only "$(get_smart_attr_raw "$output" "Command_Timeout")")
    crc=$(num_only "$(get_smart_attr_raw "$output" "UDMA_CRC_Error_Count")")

    [ -n "$reallocated" ] && echo "重分配扇区 Reallocated_Sector_Ct: $reallocated" | tee -a "$log_file"
    [ -n "$pending" ] && echo "待处理扇区 Current_Pending_Sector: $pending" | tee -a "$log_file"
    [ -n "$offline_uncorrect" ] && echo "离线不可校正 Offline_Uncorrectable: $offline_uncorrect" | tee -a "$log_file"
    [ -n "$reported_uncorrect" ] && echo "已报告不可校正 Reported_Uncorrect: $reported_uncorrect" | tee -a "$log_file"
    [ -n "$end_to_end" ] && echo "端到端错误 End-to-End_Error: $end_to_end" | tee -a "$log_file"
    [ -n "$cmd_timeout" ] && echo "命令超时 Command_Timeout: $cmd_timeout" | tee -a "$log_file"
    [ -n "$crc" ] && echo "接口 CRC 错误 UDMA_CRC_Error_Count: $crc" | tee -a "$log_file"

    if [ -n "$reallocated" ] && [ "$reallocated" -gt 0 ] 2>/dev/null; then
        echo "!!! 发现重分配扇区，磁盘可能已有坏道替换 !!!" | tee -a "$log_file"
        issue=1
    fi
    if [ -n "$pending" ] && [ "$pending" -gt 0 ] 2>/dev/null; then
        echo "!!! 发现待处理扇区，存在不稳定扇区风险 !!!" | tee -a "$log_file"
        issue=1
    fi
    if [ -n "$offline_uncorrect" ] && [ "$offline_uncorrect" -gt 0 ] 2>/dev/null; then
        echo "!!! 发现离线不可校正扇区 !!!" | tee -a "$log_file"
        issue=1
    fi
    if [ -n "$reported_uncorrect" ] && [ "$reported_uncorrect" -gt 0 ] 2>/dev/null; then
        echo "!!! 发现已报告不可校正错误 !!!" | tee -a "$log_file"
        issue=1
    fi
    if [ -n "$end_to_end" ] && [ "$end_to_end" -gt 0 ] 2>/dev/null; then
        echo "!!! 发现端到端错误，企业盘/阵列环境需重点关注 !!!" | tee -a "$log_file"
        issue=1
    fi
    if [ -n "$cmd_timeout" ] && [ "$cmd_timeout" -gt 0 ] 2>/dev/null; then
        echo ">>> 提示：发现命令超时记录，可能与硬盘、背板、线缆或 RAID 卡有关" | tee -a "$log_file"
        issue=1
    fi
    if [ -n "$crc" ] && [ "$crc" -gt 0 ] 2>/dev/null; then
        echo ">>> 提示：发现接口 CRC 错误，常见于 SATA 线缆、背板或链路问题" | tee -a "$log_file"
    fi

    # SAS 常见关键字段
    local grown_defects non_medium
    grown_defects=$(num_only "$(echo "$output" | awk -F: '/Elements in grown defect list/ {print $2; exit}')")
    non_medium=$(num_only "$(echo "$output" | awk -F: '/Non-medium error count/ {print $2; exit}')")

    [ -n "$grown_defects" ] && echo "SAS 增长缺陷表 Elements in grown defect list: $grown_defects" | tee -a "$log_file"
    [ -n "$non_medium" ] && echo "SAS 非介质错误 Non-medium error count: $non_medium" | tee -a "$log_file"

    if [ -n "$grown_defects" ] && [ "$grown_defects" -gt 0 ] 2>/dev/null; then
        echo "!!! SAS 盘增长缺陷表不为 0，存在介质缺陷增长风险 !!!" | tee -a "$log_file"
        issue=1
    fi
    if [ -n "$non_medium" ] && [ "$non_medium" -gt 0 ] 2>/dev/null; then
        echo ">>> 提示：SAS 非介质错误不为 0，可能与链路、背板、控制器有关" | tee -a "$log_file"
    fi

    # SMART 错误日志
    local errors
    errors=$(echo "$output" | grep -iE "Error [0-9]+ occurred at disk power-on lifetime" | head -n 3)
    if [ -n "$errors" ]; then
        echo ">>> 发现 ATA SMART 错误日志:" | tee -a "$log_file"
        echo "$errors" | tee -a "$log_file"
        echo "!!! 该磁盘存在历史错误日志，请结合增长趋势判断，建议重点检查 !!!" | tee -a "$log_file"
        issue=1
    fi

    if [ -n "$temp" ] && [ "$temp" -ge "$DISK_TEMP_CRIT" ] 2>/dev/null; then
        echo "!!! 磁盘温度 ${temp}℃ 过高，达到严重阈值 ${DISK_TEMP_CRIT}℃ !!!" | tee -a "$log_file"
        issue=1
    elif [ -n "$temp" ] && [ "$temp" -ge "$DISK_TEMP_WARN" ] 2>/dev/null; then
        echo ">>> 磁盘温度 ${temp}℃ 偏高，达到预警阈值 ${DISK_TEMP_WARN}℃" | tee -a "$log_file"
        issue=1
    fi

    if [ "$issue" -eq 0 ]; then
        echo "磁盘 SMART 状态正常" | tee -a "$log_file"
    fi

    return "$issue"
}

# ======================== SATA/SAS 直通/JBOD 检测 ========================
get_direct_disks() {
    # 包含 SATA/SAS/SCSI 直通盘、HBA/JBOD 盘、可能的 RAID 虚拟盘；排除 NVMe、光驱、loop、ram、fd。
    lsblk -d -n -o NAME,TYPE 2>/dev/null | awk '
        $2 == "disk" &&
        $1 !~ /^nvme[0-9]+n[0-9]+$/ &&
        $1 !~ /^loop/ &&
        $1 !~ /^sr/ &&
        $1 !~ /^ram/ &&
        $1 !~ /^fd/ {
            print $1
        }'
}

check_direct_disk_health() {
    local disk=$1
    local device="/dev/$disk"
    local output

    print_line "$DISK_LOG"
    echo "检测 SATA/SAS/直通/JBOD/可能的 RAID 虚拟盘: $device" | tee -a "$DISK_LOG"

    output=$(smartctl -a "$device" 2>&1)

    # 部分 SATA 盘经由特殊控制器时需要 -d sat
    if echo "$output" | grep -qiE "Please specify device type|Unable to detect device type"; then
        local output_sat
        output_sat=$(smartctl -a -d sat "$device" 2>&1)
        if ! echo "$output_sat" | grep -qiE "Please specify device type|Unable to detect device type|Smartctl open device"; then
            output="$output_sat"
        fi
    fi

    if echo "$output" | grep -qiE "Permission denied|Operation not permitted"; then
        echo "!!! 无权限读取 SMART 信息，请使用 root 执行脚本 !!!" | tee -a "$DISK_LOG"
        bad_disks+=("$device")
        return 1
    fi

    if echo "$output" | grep -qiE "Smartctl open device.*failed|No such device|Unable to detect device type"; then
        echo "警告：无法直接读取 $device 的 SMART 信息。若该盘是 RAID 虚拟盘，请以 RAID 物理盘检测结果为准。" | tee -a "$DISK_LOG"
        echo "$output" | head -n 8 | tee -a "$DISK_LOG"
        return 0
    fi

    analyze_ata_sas_smart_output "$device" "$DISK_LOG" "$output"
    local rc=$?
    if [ "$rc" -ne 0 ]; then
        bad_disks+=("$device")
    fi
    return "$rc"
}

# ======================== NVMe 检测 ========================
get_nvme_disks() {
    lsblk -d -n -o NAME,TYPE 2>/dev/null | awk '$2=="disk" && $1 ~ /^nvme[0-9]+n[0-9]+$/ {print $1}'
}

check_nvme_health() {
    local nvme=$1
    local device="/dev/$nvme"
    local output
    output=$(smartctl -a "$device" 2>&1)

    print_line "$NVME_LOG"
    echo "检测 NVMe 硬盘: $device" | tee -a "$NVME_LOG"

    if echo "$output" | grep -qiE "Permission denied|Operation not permitted"; then
        echo "!!! 无权限读取 NVMe SMART 信息，请使用 root 执行脚本 !!!" | tee -a "$NVME_LOG"
        bad_nvmes+=("$device")
        return 1
    fi

    if echo "$output" | grep -qiE "No such device|Unable to detect device type|Smartctl open device.*failed"; then
        echo "!!! 无法读取 NVMe SMART 信息: $device" | tee -a "$NVME_LOG"
        echo "$output" | head -n 8 | tee -a "$NVME_LOG"
        bad_nvmes+=("$device")
        return 1
    fi

    local issue=0
    local model serial health critical percentage_used spare spare_threshold media_errors err_log_entries unsafe_shutdowns temp power_on_hours data_written data_read

    model=$(echo "$output" | awk -F: '/^Model Number:/ {sub(/^[ \t]+/, "", $2); print $2; exit}')
    serial=$(echo "$output" | awk -F: '/^Serial Number:/ {sub(/^[ \t]+/, "", $2); print $2; exit}')
    health=$(echo "$output" | grep -i "SMART overall-health self-assessment test result" | head -n 1)
    critical=$(echo "$output" | awk -F: '/^Critical Warning:/ {gsub(/^[ \t]+|[ \t]+$/, "", $2); print $2; exit}')
    percentage_used=$(num_only "$(echo "$output" | awk -F: '/^Percentage Used:/ {print $2; exit}')")
    spare=$(num_only "$(echo "$output" | awk -F: '/^Available Spare:/ {print $2; exit}')")
    spare_threshold=$(num_only "$(echo "$output" | awk -F: '/^Available Spare Threshold:/ {print $2; exit}')")
    media_errors=$(num_only "$(echo "$output" | awk -F: '/^Media and Data Integrity Errors:/ {print $2; exit}')")
    err_log_entries=$(num_only "$(echo "$output" | awk -F: '/^Error Information Log Entries:/ {print $2; exit}')")
    unsafe_shutdowns=$(num_only "$(echo "$output" | awk -F: '/^Unsafe Shutdowns:/ {print $2; exit}')")
    temp=$(num_only "$(echo "$output" | awk -F: '/^Temperature:/ {print $2; exit}')")
    power_on_hours=$(echo "$output" | awk -F: '/^Power On Hours:/ {gsub(/^[ \t]+|[ \t,]+$/, "", $2); print $2; exit}')
    data_written=$(echo "$output" | awk -F: '/^Data Units Written:/ {gsub(/^[ \t]+|[ \t]+$/, "", $2); print $2; exit}')
    data_read=$(echo "$output" | awk -F: '/^Data Units Read:/ {gsub(/^[ \t]+|[ \t]+$/, "", $2); print $2; exit}')

    echo "型号: ${model:-未知}" | tee -a "$NVME_LOG"
    echo "序列号: ${serial:-未知}" | tee -a "$NVME_LOG"
    [ -n "$health" ] && echo "SMART 健康状态: $health" | tee -a "$NVME_LOG"
    echo "Critical Warning: ${critical:-未知}" | tee -a "$NVME_LOG"
    echo "磨损百分比 Percentage Used: ${percentage_used:-未知}%" | tee -a "$NVME_LOG"
    echo "可用备用空间 Available Spare: ${spare:-未知}%" | tee -a "$NVME_LOG"
    echo "备用空间阈值 Available Spare Threshold: ${spare_threshold:-未知}%" | tee -a "$NVME_LOG"
    echo "温度 Temperature: ${temp:-未知}℃" | tee -a "$NVME_LOG"
    echo "介质/数据完整性错误 Media Errors: ${media_errors:-未知}" | tee -a "$NVME_LOG"
    echo "错误日志条目 Error Log Entries: ${err_log_entries:-未知}" | tee -a "$NVME_LOG"
    echo "异常断电 Unsafe Shutdowns: ${unsafe_shutdowns:-未知}" | tee -a "$NVME_LOG"
    echo "通电时间 Power On Hours: ${power_on_hours:-未知}" | tee -a "$NVME_LOG"
    echo "累计读取 Data Units Read: ${data_read:-未知}" | tee -a "$NVME_LOG"
    echo "累计写入 Data Units Written: ${data_written:-未知}" | tee -a "$NVME_LOG"

    if [ -n "$health" ] && ! echo "$health" | grep -qi "PASSED"; then
        echo "!!! NVMe SMART 整体健康状态异常 !!!" | tee -a "$NVME_LOG"
        issue=1
    fi
    if [ -n "$critical" ] && ! echo "$critical" | grep -qiE '^(0x00|0)$'; then
        echo "!!! NVMe Critical Warning 非 0，请立即关注 !!!" | tee -a "$NVME_LOG"
        issue=1
    fi
    if [ -n "$percentage_used" ] && [ "$percentage_used" -ge "$NVME_WEAR_CRIT" ] 2>/dev/null; then
        echo "!!! NVMe 磨损已达到或超过 ${NVME_WEAR_CRIT}%，建议尽快更换 !!!" | tee -a "$NVME_LOG"
        issue=1
    elif [ -n "$percentage_used" ] && [ "$percentage_used" -ge "$NVME_WEAR_WARN" ] 2>/dev/null; then
        echo ">>> NVMe 磨损已达到 ${percentage_used}%，接近寿命上限，建议规划更换" | tee -a "$NVME_LOG"
        issue=1
    fi
    if [ -n "$spare" ] && [ -n "$spare_threshold" ] && [ "$spare" -le "$spare_threshold" ] 2>/dev/null; then
        echo "!!! NVMe 可用备用空间 ${spare}% 已低于/等于阈值 ${spare_threshold}% !!!" | tee -a "$NVME_LOG"
        issue=1
    fi
    if [ -n "$media_errors" ] && [ "$media_errors" -gt 0 ] 2>/dev/null; then
        echo "!!! NVMe 存在介质或数据完整性错误: $media_errors !!!" | tee -a "$NVME_LOG"
        issue=1
    fi
    if [ -n "$temp" ] && [ "$temp" -ge "$NVME_TEMP_CRIT" ] 2>/dev/null; then
        echo "!!! NVMe 温度 ${temp}℃ 过高，达到严重阈值 ${NVME_TEMP_CRIT}℃ !!!" | tee -a "$NVME_LOG"
        issue=1
    elif [ -n "$temp" ] && [ "$temp" -ge "$NVME_TEMP_WARN" ] 2>/dev/null; then
        echo ">>> NVMe 温度 ${temp}℃ 偏高，达到预警阈值 ${NVME_TEMP_WARN}℃" | tee -a "$NVME_LOG"
        issue=1
    fi
    if [ -n "$err_log_entries" ] && [ "$err_log_entries" -gt 0 ] 2>/dev/null; then
        echo ">>> 提示：NVMe 有历史错误日志条目: $err_log_entries，请结合时间和增长趋势判断" | tee -a "$NVME_LOG"
    fi

    if [ "$issue" -ne 0 ]; then
        bad_nvmes+=("$device")
        return 1
    fi

    echo "NVMe 状态正常" | tee -a "$NVME_LOG"
    return 0
}

# ======================== RAID 卡物理盘检测 ========================
find_storcli_tool() {
    local t
    for t in storcli64 storcli perccli64 perccli; do
        if command -v "$t" >/dev/null 2>&1; then
            command -v "$t"
            return 0
        fi
    done

    local p
    for p in \
        "/opt/MegaRAID/storcli/storcli64" \
        "/opt/MegaRAID/storcli/storcli" \
        "/opt/MegaRAID/perccli/perccli64" \
        "/opt/MegaRAID/perccli/perccli" \
        "/opt/perccli/perccli64" \
        "/opt/perccli/perccli" \
        "/usr/sbin/storcli64" \
        "/usr/bin/storcli64" \
        "/usr/sbin/storcli" \
        "/usr/bin/storcli" \
        "/usr/sbin/perccli64" \
        "/usr/bin/perccli64" \
        "/usr/sbin/perccli" \
        "/usr/bin/perccli"; do
        if [ -x "$p" ]; then
            echo "$p"
            return 0
        fi
    done
    return 1
}

find_megacli_tool() {
    local t
    for t in MegaCli64 MegaCli; do
        if command -v "$t" >/dev/null 2>&1; then
            command -v "$t"
            return 0
        fi
    done

    local p
    for p in \
        "/opt/MegaRAID/MegaCli/MegaCli64" \
        "/opt/MegaRAID/MegaCli/MegaCli" \
        "/usr/sbin/MegaCli64" \
        "/usr/bin/MegaCli64" \
        "/usr/sbin/MegaCli" \
        "/usr/bin/MegaCli"; do
        if [ -x "$p" ]; then
            echo "$p"
            return 0
        fi
    done
    return 1
}

get_storcli_controllers() {
    local storcli=$1
    local c
    for c in $(seq 0 15); do
        if "$storcli" "/c${c}" show >/tmp/storcli_c${c}_${TS}.tmp 2>&1; then
            if grep -qiE "Controller|Product Name|Status" "/tmp/storcli_c${c}_${TS}.tmp"; then
                echo "$c"
            fi
        fi
        rm -f "/tmp/storcli_c${c}_${TS}.tmp"
    done
}

check_megaraid_smart_by_did() {
    local did=$1
    local label=$2
    local log_file=$3
    local base_dev
    local output

    base_dev=$(get_first_block_disk_device)
    if [ -z "$base_dev" ]; then
        echo "提示：未找到可用于 smartctl -d megaraid,$did 的基础设备，跳过物理盘 SMART 透传检测" | tee -a "$log_file"
        return 0
    fi

    output=$(smartctl -a -d "megaraid,$did" "$base_dev" 2>&1)
    if echo "$output" | grep -qiE "Smartctl open device.*failed|INQUIRY failed|Read Device Identity failed|No such device|Unable to detect device type"; then
        echo "提示：无法通过 $base_dev 读取 megaraid,$did 的 SMART 透传信息，已保留 RAID 工具检测结果" | tee -a "$log_file"
        echo "$output" | head -n 5 | tee -a "$log_file"
        return 0
    fi

    echo "--- smartctl 透传检测：smartctl -a -d megaraid,$did $base_dev ---" | tee -a "$log_file"
    analyze_ata_sas_smart_output "$label / megaraid,$did" "$log_file" "$output"
    return $?
}

check_storcli_physical_disks() {
    local storcli=$1
    local raid_issues=0
    local controllers controller
    controllers=$(get_storcli_controllers "$storcli")

    if [ -z "$controllers" ]; then
        echo "未发现 storcli/perccli 可识别的 RAID 控制器" | tee -a "$RAID_LOG"
        return 2
    fi

    echo "使用 RAID 工具: $storcli" | tee -a "$RAID_LOG"

    for controller in $controllers; do
        print_line "$RAID_LOG"
        echo "检测 RAID 控制器: /c${controller}" | tee -a "$RAID_LOG"
        "$storcli" "/c${controller}" show | grep -iE "Product Name|Serial Number|FW Package Build|Controller Status|Status" | tee -a "$RAID_LOG" >/dev/null

        local list
        list=$("$storcli" "/c${controller}" "/eall" "/sall" show 2>/dev/null | awk '$1 ~ /^[0-9]+:[0-9]+$/ {print $1"|"$2"|"$3}')

        if [ -z "$list" ]; then
            echo "未枚举到 /c${controller} 后端物理盘，可能无硬盘、权限不足或工具版本输出格式不同" | tee -a "$RAID_LOG"
            continue
        fi

        local row eidslt eid slot did state detail model sn media other predict smart_alert intf med issue label
        while IFS='|' read -r eidslt did state; do
            [ -z "$eidslt" ] && continue
            eid=${eidslt%%:*}
            slot=${eidslt##*:}
            issue=0
            label="RAID /c${controller}/e${eid}/s${slot} DID=${did} State=${state}"

            print_line "$RAID_LOG"
            echo "检测 RAID 物理盘: $label" | tee -a "$RAID_LOG"

            detail=$("$storcli" "/c${controller}/e${eid}/s${slot}" show all 2>&1)
            model=$(echo "$detail" | awk -F= '/Model Number|Model =|Inquiry Data/ {gsub(/^[ \t]+|[ \t]+$/, "", $2); print $2; exit}')
            sn=$(echo "$detail" | awk -F= '/SN =|Serial Number/ {gsub(/^[ \t]+|[ \t]+$/, "", $2); print $2; exit}')
            media=$(num_only "$(echo "$detail" | awk -F= '/Media Error Count/ {print $2; exit}')")
            other=$(num_only "$(echo "$detail" | awk -F= '/Other Error Count/ {print $2; exit}')")
            predict=$(num_only "$(echo "$detail" | awk -F= '/Predictive Failure Count/ {print $2; exit}')")
            smart_alert=$(echo "$detail" | awk -F= '/S.M.A.R.T alert flagged by drive|SMART alert flagged by drive/ {gsub(/^[ \t]+|[ \t]+$/, "", $2); print $2; exit}')
            intf=$(echo "$detail" | awk -F= '/Intf|Interface Type/ {gsub(/^[ \t]+|[ \t]+$/, "", $2); print $2; exit}')
            med=$(echo "$detail" | awk -F= '/Med|Media Type/ {gsub(/^[ \t]+|[ \t]+$/, "", $2); print $2; exit}')

            echo "型号: ${model:-未知}" | tee -a "$RAID_LOG"
            echo "序列号: ${sn:-未知}" | tee -a "$RAID_LOG"
            echo "接口: ${intf:-未知}" | tee -a "$RAID_LOG"
            echo "介质: ${med:-未知}" | tee -a "$RAID_LOG"
            echo "RAID 状态 State: ${state:-未知}" | tee -a "$RAID_LOG"
            echo "Media Error Count: ${media:-未知}" | tee -a "$RAID_LOG"
            echo "Other Error Count: ${other:-未知}" | tee -a "$RAID_LOG"
            echo "Predictive Failure Count: ${predict:-未知}" | tee -a "$RAID_LOG"
            echo "S.M.A.R.T Alert: ${smart_alert:-未知}" | tee -a "$RAID_LOG"

            case "$state" in
                Onln|UGood|Jbod|JBOD|GHS|DHS)
                    ;;
                *)
                    echo "!!! RAID 物理盘状态不是正常/热备/JBOD: $state !!!" | tee -a "$RAID_LOG"
                    issue=1
                    ;;
            esac

            if [ -n "$media" ] && [ "$media" -gt 0 ] 2>/dev/null; then
                echo "!!! RAID 物理盘 Media Error Count 不为 0 !!!" | tee -a "$RAID_LOG"
                issue=1
            fi
            if [ -n "$other" ] && [ "$other" -gt 0 ] 2>/dev/null; then
                echo "!!! RAID 物理盘 Other Error Count 不为 0，可能有链路/背板/控制器问题 !!!" | tee -a "$RAID_LOG"
                issue=1
            fi
            if [ -n "$predict" ] && [ "$predict" -gt 0 ] 2>/dev/null; then
                echo "!!! RAID 物理盘 Predictive Failure Count 不为 0，建议尽快更换 !!!" | tee -a "$RAID_LOG"
                issue=1
            fi
            if [ -n "$smart_alert" ] && echo "$smart_alert" | grep -qiE "yes|true|1"; then
                echo "!!! RAID 物理盘已触发 SMART Alert !!!" | tee -a "$RAID_LOG"
                issue=1
            fi

            # RAID 工具检测后，再尝试 smartctl 透传读取 SATA/SAS 原始 SMART。
            check_megaraid_smart_by_did "$did" "$label" "$RAID_LOG"
            if [ $? -ne 0 ]; then
                issue=1
            fi

            if [ "$issue" -ne 0 ]; then
                bad_raids+=("/c${controller}/e${eid}/s${slot},DID=${did},State=${state}")
                raid_issues=1
            else
                echo "RAID 物理盘状态正常" | tee -a "$RAID_LOG"
            fi
        done <<EOF_STORCLI
$list
EOF_STORCLI
    done

    return "$raid_issues"
}

check_megacli_physical_disks() {
    local megacli=$1
    local raid_issues=0
    local output

    output=$("$megacli" -PDList -aALL 2>&1)
    if ! echo "$output" | grep -qi "Enclosure Device ID"; then
        echo "MegaCli 未枚举到物理盘，或当前系统无 MegaRAID 控制器" | tee -a "$RAID_LOG"
        return 2
    fi

    echo "使用 RAID 工具: $megacli" | tee -a "$RAID_LOG"

    local parsed
    parsed=$(echo "$output" | awk '
        /^Enclosure Device ID/ {eid=$NF}
        /^Slot Number/ {slot=$NF}
        /^Device Id/ {did=$NF}
        /^Firmware state/ {
            state=$0
            sub(/^Firmware state:[ \t]*/, "", state)
        }
        /^Inquiry Data/ {
            inquiry=$0
            sub(/^Inquiry Data:[ \t]*/, "", inquiry)
        }
        /^Media Error Count/ {media=$NF}
        /^Other Error Count/ {other=$NF}
        /^Predictive Failure Count/ {
            predict=$NF
            print eid "|" slot "|" did "|" state "|" inquiry "|" media "|" other "|" predict
            eid=slot=did=state=inquiry=media=other=predict=""
        }')

    if [ -z "$parsed" ]; then
        echo "MegaCli 输出存在，但脚本未能解析物理盘字段，请检查 MegaCli 版本输出格式" | tee -a "$RAID_LOG"
        return 2
    fi

    local eid slot did state inquiry media other predict issue label smart_rc
    while IFS='|' read -r eid slot did state inquiry media other predict; do
        [ -z "$eid" ] && [ -z "$slot" ] && continue
        issue=0
        label="MegaRAID EID=${eid} Slot=${slot} DID=${did} State=${state}"

        print_line "$RAID_LOG"
        echo "检测 RAID 物理盘: $label" | tee -a "$RAID_LOG"
        echo "Inquiry Data: ${inquiry:-未知}" | tee -a "$RAID_LOG"
        echo "Media Error Count: ${media:-未知}" | tee -a "$RAID_LOG"
        echo "Other Error Count: ${other:-未知}" | tee -a "$RAID_LOG"
        echo "Predictive Failure Count: ${predict:-未知}" | tee -a "$RAID_LOG"

        if ! echo "$state" | grep -qiE "Online|Unconfigured\(good\)|JBOD|Hotspare"; then
            echo "!!! RAID 物理盘状态异常: $state !!!" | tee -a "$RAID_LOG"
            issue=1
        fi
        if [ -n "$media" ] && [ "$media" -gt 0 ] 2>/dev/null; then
            echo "!!! RAID 物理盘 Media Error Count 不为 0 !!!" | tee -a "$RAID_LOG"
            issue=1
        fi
        if [ -n "$other" ] && [ "$other" -gt 0 ] 2>/dev/null; then
            echo "!!! RAID 物理盘 Other Error Count 不为 0，可能有链路/背板/控制器问题 !!!" | tee -a "$RAID_LOG"
            issue=1
        fi
        if [ -n "$predict" ] && [ "$predict" -gt 0 ] 2>/dev/null; then
            echo "!!! RAID 物理盘 Predictive Failure Count 不为 0，建议尽快更换 !!!" | tee -a "$RAID_LOG"
            issue=1
        fi

        if [ -n "$did" ]; then
            check_megaraid_smart_by_did "$did" "$label" "$RAID_LOG"
            smart_rc=$?
            [ "$smart_rc" -ne 0 ] && issue=1
        fi

        if [ "$issue" -ne 0 ]; then
            bad_raids+=("EID=${eid},Slot=${slot},DID=${did},State=${state}")
            raid_issues=1
        else
            echo "RAID 物理盘状态正常" | tee -a "$RAID_LOG"
        fi
    done <<EOF_MEGACLI
$parsed
EOF_MEGACLI

    return "$raid_issues"
}

check_raid_physical_disks() {
    local storcli megacli
    storcli=$(find_storcli_tool || true)
    megacli=$(find_megacli_tool || true)

    if [ -n "$storcli" ]; then
        check_storcli_physical_disks "$storcli"
        return $?
    fi

    if [ -n "$megacli" ]; then
        check_megacli_physical_disks "$megacli"
        return $?
    fi

    echo "未检测到 storcli/perccli/MegaCli，跳过 RAID 卡后端物理盘检测" | tee -a "$RAID_LOG"
    echo "如服务器使用 LSI/Broadcom/Dell PERC RAID，建议安装 storcli 或 perccli 后再执行" | tee -a "$RAID_LOG"
    return 2
}

# ======================== 内存检查 ========================
check_memory_errors() {
    print_line "$MEM_LOG"
    echo "开始内存错误检查..." | tee -a "$MEM_LOG"

    local mem_errors corrected_errors edac_errors mce_errors
    mem_errors=$(dmesg 2>/dev/null | grep -iE 'memory read error|hardware error|machine check|edac|mce' | head -n 30)
    corrected_errors=$(dmesg 2>/dev/null | grep -iE 'It has been corrected|corrected error|CE memory read error|EDAC.*CE' | head -n 20)
    edac_errors=$(dmesg 2>/dev/null | grep -iE 'EDAC.*(UE|uncorrected|uncorrectable)' | head -n 20)
    mce_errors=$(dmesg 2>/dev/null | grep -iE 'MCE|Machine Check|Hardware Error' | head -n 20)

    if [ -n "$edac_errors" ]; then
        echo ">>> 发现 EDAC 未校正/严重内存错误:" | tee -a "$MEM_LOG"
        echo "$edac_errors" | tee -a "$MEM_LOG"
        echo "!!! 发现疑似未修复内存错误，请立即检查内存/主板/CPU 内存通道 !!!" | tee -a "$MEM_LOG"
        return 2
    fi

    if [ -n "$corrected_errors" ]; then
        echo ">>> 发现已修复/可纠正内存错误:" | tee -a "$MEM_LOG"
        echo "$corrected_errors" | tee -a "$MEM_LOG"
        echo "!!! 系统已自动修复部分内存错误，请关注是否持续增长 !!!" | tee -a "$MEM_LOG"
        return 1
    fi

    if [ -n "$mce_errors" ]; then
        echo ">>> 发现 MCE/硬件错误日志:" | tee -a "$MEM_LOG"
        echo "$mce_errors" | tee -a "$MEM_LOG"
        echo "!!! 发现硬件错误日志，请结合 mcelog/rasdaemon 进一步确认 !!!" | tee -a "$MEM_LOG"
        return 1
    fi

    if [ -n "$mem_errors" ]; then
        echo ">>> 发现疑似内存/硬件错误日志:" | tee -a "$MEM_LOG"
        echo "$mem_errors" | tee -a "$MEM_LOG"
        echo "!!! 请进一步确认是否为内存 ECC/硬件错误 !!!" | tee -a "$MEM_LOG"
        return 1
    fi

    echo "未发现内存读取/ECC/MCE 错误" | tee -a "$MEM_LOG"
    return 0
}

# ======================== 主流程 ========================
echo "开始系统硬件健康检测..."
echo "----------------------------------------"

if ! need_cmd smartctl; then
    echo "错误：smartctl 工具未安装，请先安装 smartmontools 包"
    echo "CentOS/RHEL: yum install smartmontools -y"
    echo "Ubuntu/Debian: apt-get install smartmontools -y"
    exit 1
fi

# SATA/SAS/直通/JBOD/RAID 虚拟盘检测
echo ""
echo "========== SATA/SAS/直通/JBOD 磁盘健康检查 =========="
disk_issues=0
DIRECT_DISKS=$(get_direct_disks)
if [ -z "$DIRECT_DISKS" ]; then
    echo "未检测到 SATA/SAS/SCSI 类型块设备"
    disk_status="未检测到 SATA/SAS/SCSI 类型块设备"
else
    for disk in $DIRECT_DISKS; do
        check_direct_disk_health "$disk"
        [ $? -ne 0 ] && disk_issues=1
        echo ""
    done

    if [ "$disk_issues" -eq 1 ]; then
        echo "警告：检测到 SATA/SAS/直通/JBOD 磁盘存在问题，请查看详细日志：$DISK_LOG"
        disk_status="存在问题"
    else
        echo "SATA/SAS/直通/JBOD 磁盘状态正常或未能直接读取的 RAID 虚拟盘已跳过故障判定"
        disk_status="状态良好"
    fi
fi

# NVMe 检测
echo ""
echo "========== NVMe 磨损/健康检查 =========="
nvme_issues=0
NVMES=$(get_nvme_disks)
if [ -z "$NVMES" ]; then
    echo "未检测到 NVMe 硬盘"
    nvme_status="未检测到 NVMe 硬盘"
else
    for nvme in $NVMES; do
        check_nvme_health "$nvme"
        [ $? -ne 0 ] && nvme_issues=1
        echo ""
    done

    if [ "$nvme_issues" -eq 1 ]; then
        echo "警告：检测到 NVMe 存在磨损或健康问题，请查看详细日志：$NVME_LOG"
        nvme_status="存在问题"
    else
        echo "所有 NVMe 状态正常"
        nvme_status="状态良好"
    fi
fi

# RAID 后端物理盘检测
echo ""
echo "========== RAID 卡后端物理盘/JBOD 检查 =========="
raid_issues=0
raid_result=0
check_raid_physical_disks
raid_result=$?
case "$raid_result" in
    0)
        raid_status="状态良好"
        echo "RAID 后端物理盘状态正常，详细日志：$RAID_LOG"
        ;;
    1)
        raid_issues=1
        raid_status="存在问题"
        echo "警告：检测到 RAID 后端物理盘存在问题，请查看详细日志：$RAID_LOG"
        ;;
    2)
        raid_status="未检测或未安装 RAID 工具"
        echo "RAID 后端物理盘检测已跳过，详细日志：$RAID_LOG"
        ;;
esac

# 内存检测
echo ""
echo "========== 内存健康检查 =========="
check_memory_errors
mem_status=$?
case "$mem_status" in
    0)
        echo "内存状态正常"
        mem_result="状态良好"
        ;;
    1)
        echo "警告：检测到已修复或疑似硬件错误，请查看详细日志：$MEM_LOG"
        mem_result="存在已修复/疑似错误"
        ;;
    2)
        echo "严重：检测到未修复的内存错误，请立即检查内存！查看日志：$MEM_LOG"
        mem_result="存在未修复错误"
        ;;
    *)
        echo "内存检测状态未知"
        mem_result="状态未知"
        ;;
esac

# 汇总
echo ""
echo "========== 检测总结 =========="
echo "SATA/SAS/直通/JBOD 磁盘状态: $disk_status"
if [ ${#bad_disks[@]} -gt 0 ]; then
    echo "有问题的 SATA/SAS/直通/JBOD 磁盘: ${bad_disks[*]}"
fi

echo "NVMe 状态: $nvme_status"
if [ ${#bad_nvmes[@]} -gt 0 ]; then
    echo "有问题的 NVMe: ${bad_nvmes[*]}"
fi

echo "RAID 后端物理盘状态: $raid_status"
if [ ${#bad_raids[@]} -gt 0 ]; then
    echo "有问题的 RAID 物理盘: ${bad_raids[*]}"
fi

echo "内存状态: $mem_result"

if [ -n "$DIRECT_DISKS" ]; then
    echo "SATA/SAS/SCSI 块设备数量: $(echo "$DIRECT_DISKS" | wc -w)"
fi
if [ -n "$NVMES" ]; then
    echo "NVMe 数量: $(echo "$NVMES" | wc -w)"
fi

echo ""
echo "检测完成"
echo "详细日志已保存至:"
echo "  磁盘日志: $DISK_LOG"
echo "  NVMe 日志: $NVME_LOG"
echo "  RAID 日志: $RAID_LOG"
echo "  内存日志: $MEM_LOG"

# 返回整体检测状态码：RAID 工具未安装不算失败；检测到实际硬件异常才返回 1。
if [ "$disk_issues" -eq 1 ] || [ "$nvme_issues" -eq 1 ] || [ "$raid_issues" -eq 1 ] || [ "$mem_status" -ne 0 ]; then
    exit 1
else
    exit 0
fi