From 39711cf6c2ec5a3b4480dcb4800cc3802bda5bf2 Mon Sep 17 00:00:00 2001 From: Lukasz Kasprzak Date: Fri, 20 Mar 2026 19:16:32 +0100 Subject: feat: first fully successful run e2e on straper --- straper/db/public/zfs/etc-zfs/zpool.d/ata_err | 249 +++++++++++++++++++++ straper/db/public/zfs/etc-zfs/zpool.d/cmd_to | 249 +++++++++++++++++++++ straper/db/public/zfs/etc-zfs/zpool.d/defect | 249 +++++++++++++++++++++ straper/db/public/zfs/etc-zfs/zpool.d/dm-deps | 28 +++ straper/db/public/zfs/etc-zfs/zpool.d/enc | 69 ++++++ straper/db/public/zfs/etc-zfs/zpool.d/encdev | 69 ++++++ straper/db/public/zfs/etc-zfs/zpool.d/fault_led | 69 ++++++ straper/db/public/zfs/etc-zfs/zpool.d/health | 249 +++++++++++++++++++++ straper/db/public/zfs/etc-zfs/zpool.d/hours_on | 249 +++++++++++++++++++++ straper/db/public/zfs/etc-zfs/zpool.d/iostat | 86 +++++++ straper/db/public/zfs/etc-zfs/zpool.d/iostat-10s | 86 +++++++ straper/db/public/zfs/etc-zfs/zpool.d/iostat-1s | 86 +++++++ straper/db/public/zfs/etc-zfs/zpool.d/label | 92 ++++++++ straper/db/public/zfs/etc-zfs/zpool.d/locate_led | 69 ++++++ straper/db/public/zfs/etc-zfs/zpool.d/lsblk | 92 ++++++++ straper/db/public/zfs/etc-zfs/zpool.d/media | 33 +++ straper/db/public/zfs/etc-zfs/zpool.d/model | 92 ++++++++ straper/db/public/zfs/etc-zfs/zpool.d/nonmed | 249 +++++++++++++++++++++ straper/db/public/zfs/etc-zfs/zpool.d/nvme_err | 249 +++++++++++++++++++++ straper/db/public/zfs/etc-zfs/zpool.d/off_ucor | 249 +++++++++++++++++++++ straper/db/public/zfs/etc-zfs/zpool.d/pend_sec | 249 +++++++++++++++++++++ straper/db/public/zfs/etc-zfs/zpool.d/pwr_cyc | 249 +++++++++++++++++++++ straper/db/public/zfs/etc-zfs/zpool.d/r_proc | 249 +++++++++++++++++++++ straper/db/public/zfs/etc-zfs/zpool.d/r_ucor | 249 +++++++++++++++++++++ straper/db/public/zfs/etc-zfs/zpool.d/realloc | 249 +++++++++++++++++++++ straper/db/public/zfs/etc-zfs/zpool.d/rep_ucor | 249 +++++++++++++++++++++ straper/db/public/zfs/etc-zfs/zpool.d/serial | 249 +++++++++++++++++++++ straper/db/public/zfs/etc-zfs/zpool.d/ses | 69 ++++++ straper/db/public/zfs/etc-zfs/zpool.d/size | 92 ++++++++ straper/db/public/zfs/etc-zfs/zpool.d/slot | 69 ++++++ straper/db/public/zfs/etc-zfs/zpool.d/smart | 249 +++++++++++++++++++++ straper/db/public/zfs/etc-zfs/zpool.d/smart_test | 249 +++++++++++++++++++++ straper/db/public/zfs/etc-zfs/zpool.d/smartx | 249 +++++++++++++++++++++ straper/db/public/zfs/etc-zfs/zpool.d/temp | 249 +++++++++++++++++++++ straper/db/public/zfs/etc-zfs/zpool.d/test_ended | 249 +++++++++++++++++++++ .../db/public/zfs/etc-zfs/zpool.d/test_progress | 249 +++++++++++++++++++++ straper/db/public/zfs/etc-zfs/zpool.d/test_status | 249 +++++++++++++++++++++ straper/db/public/zfs/etc-zfs/zpool.d/test_type | 249 +++++++++++++++++++++ straper/db/public/zfs/etc-zfs/zpool.d/upath | 8 + straper/db/public/zfs/etc-zfs/zpool.d/vendor | 92 ++++++++ straper/db/public/zfs/etc-zfs/zpool.d/w_proc | 249 +++++++++++++++++++++ straper/db/public/zfs/etc-zfs/zpool.d/w_ucor | 249 +++++++++++++++++++++ 42 files changed, 7426 insertions(+) create mode 100755 straper/db/public/zfs/etc-zfs/zpool.d/ata_err create mode 100755 straper/db/public/zfs/etc-zfs/zpool.d/cmd_to create mode 100755 straper/db/public/zfs/etc-zfs/zpool.d/defect create mode 100755 straper/db/public/zfs/etc-zfs/zpool.d/dm-deps create mode 100755 straper/db/public/zfs/etc-zfs/zpool.d/enc create mode 100755 straper/db/public/zfs/etc-zfs/zpool.d/encdev create mode 100755 straper/db/public/zfs/etc-zfs/zpool.d/fault_led create mode 100755 straper/db/public/zfs/etc-zfs/zpool.d/health create mode 100755 straper/db/public/zfs/etc-zfs/zpool.d/hours_on create mode 100755 straper/db/public/zfs/etc-zfs/zpool.d/iostat create mode 100755 straper/db/public/zfs/etc-zfs/zpool.d/iostat-10s create mode 100755 straper/db/public/zfs/etc-zfs/zpool.d/iostat-1s create mode 100755 straper/db/public/zfs/etc-zfs/zpool.d/label create mode 100755 straper/db/public/zfs/etc-zfs/zpool.d/locate_led create mode 100755 straper/db/public/zfs/etc-zfs/zpool.d/lsblk create mode 100755 straper/db/public/zfs/etc-zfs/zpool.d/media create mode 100755 straper/db/public/zfs/etc-zfs/zpool.d/model create mode 100755 straper/db/public/zfs/etc-zfs/zpool.d/nonmed create mode 100755 straper/db/public/zfs/etc-zfs/zpool.d/nvme_err create mode 100755 straper/db/public/zfs/etc-zfs/zpool.d/off_ucor create mode 100755 straper/db/public/zfs/etc-zfs/zpool.d/pend_sec create mode 100755 straper/db/public/zfs/etc-zfs/zpool.d/pwr_cyc create mode 100755 straper/db/public/zfs/etc-zfs/zpool.d/r_proc create mode 100755 straper/db/public/zfs/etc-zfs/zpool.d/r_ucor create mode 100755 straper/db/public/zfs/etc-zfs/zpool.d/realloc create mode 100755 straper/db/public/zfs/etc-zfs/zpool.d/rep_ucor create mode 100755 straper/db/public/zfs/etc-zfs/zpool.d/serial create mode 100755 straper/db/public/zfs/etc-zfs/zpool.d/ses create mode 100755 straper/db/public/zfs/etc-zfs/zpool.d/size create mode 100755 straper/db/public/zfs/etc-zfs/zpool.d/slot create mode 100755 straper/db/public/zfs/etc-zfs/zpool.d/smart create mode 100755 straper/db/public/zfs/etc-zfs/zpool.d/smart_test create mode 100755 straper/db/public/zfs/etc-zfs/zpool.d/smartx create mode 100755 straper/db/public/zfs/etc-zfs/zpool.d/temp create mode 100755 straper/db/public/zfs/etc-zfs/zpool.d/test_ended create mode 100755 straper/db/public/zfs/etc-zfs/zpool.d/test_progress create mode 100755 straper/db/public/zfs/etc-zfs/zpool.d/test_status create mode 100755 straper/db/public/zfs/etc-zfs/zpool.d/test_type create mode 100755 straper/db/public/zfs/etc-zfs/zpool.d/upath create mode 100755 straper/db/public/zfs/etc-zfs/zpool.d/vendor create mode 100755 straper/db/public/zfs/etc-zfs/zpool.d/w_proc create mode 100755 straper/db/public/zfs/etc-zfs/zpool.d/w_ucor (limited to 'straper/db/public/zfs/etc-zfs/zpool.d') diff --git a/straper/db/public/zfs/etc-zfs/zpool.d/ata_err b/straper/db/public/zfs/etc-zfs/zpool.d/ata_err new file mode 100755 index 0000000..8ad3e10 --- /dev/null +++ b/straper/db/public/zfs/etc-zfs/zpool.d/ata_err @@ -0,0 +1,249 @@ +#!/bin/sh +# +# Show SMART stats +# + +helpstr=" +smart: Show SMART temperature and error stats (specific to drive type) +smartx: Show SMART extended drive stats (specific to drive type). +temp: Show SMART drive temperature in celsius (all drives). +health: Show reported SMART status (all drives). +r_proc: Show SMART read GBytes processed over drive lifetime (SAS). +w_proc: Show SMART write GBytes processed over drive lifetime (SAS). +r_ucor: Show SMART read uncorrectable errors (SAS). +w_ucor: Show SMART write uncorrectable errors (SAS). +nonmed: Show SMART non-medium errors (SAS). +defect: Show SMART grown defect list (SAS). +hours_on: Show number of hours drive powered on (all drives). +realloc: Show SMART reallocated sectors count (ATA). +rep_ucor: Show SMART reported uncorrectable count (ATA). +cmd_to: Show SMART command timeout count (ATA). +pend_sec: Show SMART current pending sector count (ATA). +off_ucor: Show SMART offline uncorrectable errors (ATA). +ata_err: Show SMART ATA errors (ATA). +pwr_cyc: Show SMART power cycle count (ATA). +serial: Show disk serial number. +nvme_err: Show SMART NVMe errors (NVMe). +smart_test: Show SMART self-test results summary. +test_type: Show SMART self-test type (short, long... ). +test_status: Show SMART self-test status. +test_progress: Show SMART self-test percentage done. +test_ended: Show when the last SMART self-test ended (if supported). +" + +# Hack for developer testing +# +# If you set $samples to a directory containing smartctl output text files, +# we will use them instead of running smartctl on the vdevs. This can be +# useful if you want to test a bunch of different smartctl outputs. Also, if +# $samples is set, and additional 'file' column is added to the zpool output +# showing the filename. +samples= + +# get_filename_from_dir DIR +# +# Look in directory DIR and return a filename from it. The filename returned +# is chosen quasi-sequentially (based off our PID). This allows us to return +# a different filename every time this script is invoked (which we do for each +# vdev), without having to maintain state. +get_filename_from_dir() +{ + dir=$1 + pid="$$" + num_files=$(find "$dir" -maxdepth 1 -type f | wc -l) + mod=$((pid % num_files)) + i=0 + find "$dir" -type f -printf '%f\n' | while read -r file ; do + if [ "$mod" = "$i" ] ; then + echo "$file" + break + fi + i=$((i+1)) + done +} + +script="${0##*/}" + +if [ "$1" = "-h" ] ; then + echo "$helpstr" | grep "$script:" | tr -s '\t' | cut -f 2- + exit +fi + +# Sometimes, UPATH ends up /dev/(null). +# That should be corrected, but for now... +# shellcheck disable=SC2154 +if [ ! -b "$VDEV_UPATH" ]; then + somepath="${VDEV_PATH}" +else + somepath="${VDEV_UPATH}" +fi + +if [ -b "$somepath" ] && PATH="/usr/sbin:$PATH" command -v smartctl > /dev/null || [ -n "$samples" ] ; then + if [ -n "$samples" ] ; then + # cat a smartctl output text file instead of running smartctl + # on a vdev (only used for developer testing). + file=$(get_filename_from_dir "$samples") + echo "file=$file" + raw_out=$(cat "$samples/$file") + else + raw_out=$(sudo smartctl -a "$somepath") + fi + + # What kind of drive are we? Look for the right line in smartctl: + # + # SAS: + # Transport protocol: SAS + # + # SATA: + # ATA Version is: 8 + # + # NVMe: + # SMART/Health Information (NVMe Log 0xnn, NSID 0xnn) + # + out=$(echo "$raw_out" | awk ' +# SAS specific +/read:/{print "rrd="$4"\nr_cor="$5"\nr_proc="$7"\nr_ucor="$8} +/write:/{print "rwr="$4"\nw_cor="$5"\nw_proc="$7"\nw_ucor="$8} +/Non-medium error count/{print "nonmed="$4} +/Elements in grown defect list/{print "defect="$6} + +# SAS common +/SAS/{type="sas"} +/Drive Temperature:/{print "temp="$4} +# Status can be a long string, substitute spaces for '_' +/SMART Health Status:/{printf "health="; for(i=4;i<=NF-1;i++){printf "%s_", $i}; printf "%s\n", $i} +/number of hours powered up/{print "hours_on="$7; hours_on=int($7)} +/Serial number:/{print "serial="$3} + +# SATA specific +/Reallocated_Sector_Ct/{print "realloc="$10} +/Reported_Uncorrect/{print "rep_ucor="$10} +/Command_Timeout/{print "cmd_to="$10} +/Current_Pending_Sector/{print "pend_sec="$10} +/Offline_Uncorrectable/{print "off_ucor="$10} +/ATA Error Count:/{print "ata_err="$4} +/Power_Cycle_Count/{print "pwr_cyc="$10} + +# SATA common +/SATA/{type="sata"} +/Temperature_Celsius/{print "temp="$10} +/Airflow_Temperature_Cel/{print "temp="$10} +/Current Temperature:/{print "temp="$3} +/SMART overall-health self-assessment test result:/{print "health="$6} +/Power_On_Hours/{print "hours_on="$10; hours_on=int($10)} +/Serial Number:/{print "serial="$3} + +# NVMe common +/NVMe/{type="nvme"} +/Temperature:/{print "temp="$2} +/SMART overall-health self-assessment test result:/{print "health="$6} +/Power On Hours:/{gsub("[^0-9]","",$4); print "hours_on="$4} +/Serial Number:/{print "serial="$3} +/Power Cycles:/{print "pwr_cyc="$3} + +# NVMe specific +/Media and Data Integrity Errors:/{print "nvme_err="$6} + +# SMART self-test info +/Self-test execution status:/{progress=tolower($4)} # SAS +/SMART Self-test log/{test_seen=1} # SAS +/SMART Extended Self-test Log/{test_seen=1} # SATA +/# 1/{ + test_type=tolower($3"_"$4); + # Status could be one word ("Completed") or multiple ("Completed: read + # failure"). Look for the ":" to see if we need to grab more words. + + if ($5 ~ ":") + status=tolower($5""$6"_"$7) + else + status=tolower($5) + if (status=="self") + status="running"; + + if (type == "sas") { + hours=int($(NF-4)) + } else { + hours=int($(NF-1)) + # SATA reports percent remaining, rather than percent done + # Convert it to percent done. + progress=(100-int($(NF-2)))"%" + } + # When we int()-ify "hours", it converts stuff like "NOW" and "-" into + # 0. In those cases, set it to hours_on, so they will cancel out in + # the "hours_ago" calculation later on. + if (hours == 0) + hours=hours_on + + if (test_seen) { + print "test="hours_on + print "test_type="test_type + print "test_status="status + print "test_progress="progress + } + # Not all drives report hours_on + if (hours_on && hours) { + total_hours_ago=(hours_on-hours) + days_ago=int(total_hours_ago/24) + hours_ago=(total_hours_ago % 24) + if (days_ago != 0) + ago_str=days_ago"d" + if (hours_ago !=0) + ago_str=ago_str""hours_ago"h" + print "test_ended="ago_str + } +} + +END {print "type="type; ORS="\n"; print ""} +'); +fi +type=$(echo "$out" | grep '^type=' | cut -d '=' -f 2) + +# If type is not set by now, either we don't have a block device +# or smartctl failed. Either way, default to ATA and set $out to +# nothing. +if [ -z "$type" ]; then + type="sata" + out= +fi + +case $script in +smart) + # Print temperature plus common predictors of drive failure + if [ "$type" = "sas" ] ; then + scripts="temp|health|r_ucor|w_ucor" + elif [ "$type" = "sata" ] ; then + scripts="temp|health|ata_err|realloc|rep_ucor|cmd_to|pend_sec|off_ucor" + elif [ "$type" = "nvme" ] ; then + scripts="temp|health|nvme_err" + fi + ;; +smartx) + # Print some other interesting stats + if [ "$type" = "sas" ] ; then + scripts="hours_on|defect|nonmed|r_proc|w_proc" + elif [ "$type" = "sata" ] ; then + scripts="hours_on|pwr_cyc" + elif [ "$type" = "nvme" ] ; then + scripts="hours_on|pwr_cyc" + fi + ;; +smart_test) + scripts="test_type|test_status|test_progress|test_ended" + ;; +*) + scripts="$script" +esac + +with_vals=$(echo "$out" | grep -E "$scripts") +if [ -n "$with_vals" ]; then + echo "$with_vals" + without_vals=$(echo "$scripts" | tr '|' '\n' | + grep -v -E "$(echo "$with_vals" | + awk -F "=" '{print $1}')" | awk '{print $0"="}') +else + without_vals=$(echo "$scripts" | tr '|' '\n' | awk '{print $0"="}') +fi + +if [ -n "$without_vals" ]; then + echo "$without_vals" +fi diff --git a/straper/db/public/zfs/etc-zfs/zpool.d/cmd_to b/straper/db/public/zfs/etc-zfs/zpool.d/cmd_to new file mode 100755 index 0000000..8ad3e10 --- /dev/null +++ b/straper/db/public/zfs/etc-zfs/zpool.d/cmd_to @@ -0,0 +1,249 @@ +#!/bin/sh +# +# Show SMART stats +# + +helpstr=" +smart: Show SMART temperature and error stats (specific to drive type) +smartx: Show SMART extended drive stats (specific to drive type). +temp: Show SMART drive temperature in celsius (all drives). +health: Show reported SMART status (all drives). +r_proc: Show SMART read GBytes processed over drive lifetime (SAS). +w_proc: Show SMART write GBytes processed over drive lifetime (SAS). +r_ucor: Show SMART read uncorrectable errors (SAS). +w_ucor: Show SMART write uncorrectable errors (SAS). +nonmed: Show SMART non-medium errors (SAS). +defect: Show SMART grown defect list (SAS). +hours_on: Show number of hours drive powered on (all drives). +realloc: Show SMART reallocated sectors count (ATA). +rep_ucor: Show SMART reported uncorrectable count (ATA). +cmd_to: Show SMART command timeout count (ATA). +pend_sec: Show SMART current pending sector count (ATA). +off_ucor: Show SMART offline uncorrectable errors (ATA). +ata_err: Show SMART ATA errors (ATA). +pwr_cyc: Show SMART power cycle count (ATA). +serial: Show disk serial number. +nvme_err: Show SMART NVMe errors (NVMe). +smart_test: Show SMART self-test results summary. +test_type: Show SMART self-test type (short, long... ). +test_status: Show SMART self-test status. +test_progress: Show SMART self-test percentage done. +test_ended: Show when the last SMART self-test ended (if supported). +" + +# Hack for developer testing +# +# If you set $samples to a directory containing smartctl output text files, +# we will use them instead of running smartctl on the vdevs. This can be +# useful if you want to test a bunch of different smartctl outputs. Also, if +# $samples is set, and additional 'file' column is added to the zpool output +# showing the filename. +samples= + +# get_filename_from_dir DIR +# +# Look in directory DIR and return a filename from it. The filename returned +# is chosen quasi-sequentially (based off our PID). This allows us to return +# a different filename every time this script is invoked (which we do for each +# vdev), without having to maintain state. +get_filename_from_dir() +{ + dir=$1 + pid="$$" + num_files=$(find "$dir" -maxdepth 1 -type f | wc -l) + mod=$((pid % num_files)) + i=0 + find "$dir" -type f -printf '%f\n' | while read -r file ; do + if [ "$mod" = "$i" ] ; then + echo "$file" + break + fi + i=$((i+1)) + done +} + +script="${0##*/}" + +if [ "$1" = "-h" ] ; then + echo "$helpstr" | grep "$script:" | tr -s '\t' | cut -f 2- + exit +fi + +# Sometimes, UPATH ends up /dev/(null). +# That should be corrected, but for now... +# shellcheck disable=SC2154 +if [ ! -b "$VDEV_UPATH" ]; then + somepath="${VDEV_PATH}" +else + somepath="${VDEV_UPATH}" +fi + +if [ -b "$somepath" ] && PATH="/usr/sbin:$PATH" command -v smartctl > /dev/null || [ -n "$samples" ] ; then + if [ -n "$samples" ] ; then + # cat a smartctl output text file instead of running smartctl + # on a vdev (only used for developer testing). + file=$(get_filename_from_dir "$samples") + echo "file=$file" + raw_out=$(cat "$samples/$file") + else + raw_out=$(sudo smartctl -a "$somepath") + fi + + # What kind of drive are we? Look for the right line in smartctl: + # + # SAS: + # Transport protocol: SAS + # + # SATA: + # ATA Version is: 8 + # + # NVMe: + # SMART/Health Information (NVMe Log 0xnn, NSID 0xnn) + # + out=$(echo "$raw_out" | awk ' +# SAS specific +/read:/{print "rrd="$4"\nr_cor="$5"\nr_proc="$7"\nr_ucor="$8} +/write:/{print "rwr="$4"\nw_cor="$5"\nw_proc="$7"\nw_ucor="$8} +/Non-medium error count/{print "nonmed="$4} +/Elements in grown defect list/{print "defect="$6} + +# SAS common +/SAS/{type="sas"} +/Drive Temperature:/{print "temp="$4} +# Status can be a long string, substitute spaces for '_' +/SMART Health Status:/{printf "health="; for(i=4;i<=NF-1;i++){printf "%s_", $i}; printf "%s\n", $i} +/number of hours powered up/{print "hours_on="$7; hours_on=int($7)} +/Serial number:/{print "serial="$3} + +# SATA specific +/Reallocated_Sector_Ct/{print "realloc="$10} +/Reported_Uncorrect/{print "rep_ucor="$10} +/Command_Timeout/{print "cmd_to="$10} +/Current_Pending_Sector/{print "pend_sec="$10} +/Offline_Uncorrectable/{print "off_ucor="$10} +/ATA Error Count:/{print "ata_err="$4} +/Power_Cycle_Count/{print "pwr_cyc="$10} + +# SATA common +/SATA/{type="sata"} +/Temperature_Celsius/{print "temp="$10} +/Airflow_Temperature_Cel/{print "temp="$10} +/Current Temperature:/{print "temp="$3} +/SMART overall-health self-assessment test result:/{print "health="$6} +/Power_On_Hours/{print "hours_on="$10; hours_on=int($10)} +/Serial Number:/{print "serial="$3} + +# NVMe common +/NVMe/{type="nvme"} +/Temperature:/{print "temp="$2} +/SMART overall-health self-assessment test result:/{print "health="$6} +/Power On Hours:/{gsub("[^0-9]","",$4); print "hours_on="$4} +/Serial Number:/{print "serial="$3} +/Power Cycles:/{print "pwr_cyc="$3} + +# NVMe specific +/Media and Data Integrity Errors:/{print "nvme_err="$6} + +# SMART self-test info +/Self-test execution status:/{progress=tolower($4)} # SAS +/SMART Self-test log/{test_seen=1} # SAS +/SMART Extended Self-test Log/{test_seen=1} # SATA +/# 1/{ + test_type=tolower($3"_"$4); + # Status could be one word ("Completed") or multiple ("Completed: read + # failure"). Look for the ":" to see if we need to grab more words. + + if ($5 ~ ":") + status=tolower($5""$6"_"$7) + else + status=tolower($5) + if (status=="self") + status="running"; + + if (type == "sas") { + hours=int($(NF-4)) + } else { + hours=int($(NF-1)) + # SATA reports percent remaining, rather than percent done + # Convert it to percent done. + progress=(100-int($(NF-2)))"%" + } + # When we int()-ify "hours", it converts stuff like "NOW" and "-" into + # 0. In those cases, set it to hours_on, so they will cancel out in + # the "hours_ago" calculation later on. + if (hours == 0) + hours=hours_on + + if (test_seen) { + print "test="hours_on + print "test_type="test_type + print "test_status="status + print "test_progress="progress + } + # Not all drives report hours_on + if (hours_on && hours) { + total_hours_ago=(hours_on-hours) + days_ago=int(total_hours_ago/24) + hours_ago=(total_hours_ago % 24) + if (days_ago != 0) + ago_str=days_ago"d" + if (hours_ago !=0) + ago_str=ago_str""hours_ago"h" + print "test_ended="ago_str + } +} + +END {print "type="type; ORS="\n"; print ""} +'); +fi +type=$(echo "$out" | grep '^type=' | cut -d '=' -f 2) + +# If type is not set by now, either we don't have a block device +# or smartctl failed. Either way, default to ATA and set $out to +# nothing. +if [ -z "$type" ]; then + type="sata" + out= +fi + +case $script in +smart) + # Print temperature plus common predictors of drive failure + if [ "$type" = "sas" ] ; then + scripts="temp|health|r_ucor|w_ucor" + elif [ "$type" = "sata" ] ; then + scripts="temp|health|ata_err|realloc|rep_ucor|cmd_to|pend_sec|off_ucor" + elif [ "$type" = "nvme" ] ; then + scripts="temp|health|nvme_err" + fi + ;; +smartx) + # Print some other interesting stats + if [ "$type" = "sas" ] ; then + scripts="hours_on|defect|nonmed|r_proc|w_proc" + elif [ "$type" = "sata" ] ; then + scripts="hours_on|pwr_cyc" + elif [ "$type" = "nvme" ] ; then + scripts="hours_on|pwr_cyc" + fi + ;; +smart_test) + scripts="test_type|test_status|test_progress|test_ended" + ;; +*) + scripts="$script" +esac + +with_vals=$(echo "$out" | grep -E "$scripts") +if [ -n "$with_vals" ]; then + echo "$with_vals" + without_vals=$(echo "$scripts" | tr '|' '\n' | + grep -v -E "$(echo "$with_vals" | + awk -F "=" '{print $1}')" | awk '{print $0"="}') +else + without_vals=$(echo "$scripts" | tr '|' '\n' | awk '{print $0"="}') +fi + +if [ -n "$without_vals" ]; then + echo "$without_vals" +fi diff --git a/straper/db/public/zfs/etc-zfs/zpool.d/defect b/straper/db/public/zfs/etc-zfs/zpool.d/defect new file mode 100755 index 0000000..8ad3e10 --- /dev/null +++ b/straper/db/public/zfs/etc-zfs/zpool.d/defect @@ -0,0 +1,249 @@ +#!/bin/sh +# +# Show SMART stats +# + +helpstr=" +smart: Show SMART temperature and error stats (specific to drive type) +smartx: Show SMART extended drive stats (specific to drive type). +temp: Show SMART drive temperature in celsius (all drives). +health: Show reported SMART status (all drives). +r_proc: Show SMART read GBytes processed over drive lifetime (SAS). +w_proc: Show SMART write GBytes processed over drive lifetime (SAS). +r_ucor: Show SMART read uncorrectable errors (SAS). +w_ucor: Show SMART write uncorrectable errors (SAS). +nonmed: Show SMART non-medium errors (SAS). +defect: Show SMART grown defect list (SAS). +hours_on: Show number of hours drive powered on (all drives). +realloc: Show SMART reallocated sectors count (ATA). +rep_ucor: Show SMART reported uncorrectable count (ATA). +cmd_to: Show SMART command timeout count (ATA). +pend_sec: Show SMART current pending sector count (ATA). +off_ucor: Show SMART offline uncorrectable errors (ATA). +ata_err: Show SMART ATA errors (ATA). +pwr_cyc: Show SMART power cycle count (ATA). +serial: Show disk serial number. +nvme_err: Show SMART NVMe errors (NVMe). +smart_test: Show SMART self-test results summary. +test_type: Show SMART self-test type (short, long... ). +test_status: Show SMART self-test status. +test_progress: Show SMART self-test percentage done. +test_ended: Show when the last SMART self-test ended (if supported). +" + +# Hack for developer testing +# +# If you set $samples to a directory containing smartctl output text files, +# we will use them instead of running smartctl on the vdevs. This can be +# useful if you want to test a bunch of different smartctl outputs. Also, if +# $samples is set, and additional 'file' column is added to the zpool output +# showing the filename. +samples= + +# get_filename_from_dir DIR +# +# Look in directory DIR and return a filename from it. The filename returned +# is chosen quasi-sequentially (based off our PID). This allows us to return +# a different filename every time this script is invoked (which we do for each +# vdev), without having to maintain state. +get_filename_from_dir() +{ + dir=$1 + pid="$$" + num_files=$(find "$dir" -maxdepth 1 -type f | wc -l) + mod=$((pid % num_files)) + i=0 + find "$dir" -type f -printf '%f\n' | while read -r file ; do + if [ "$mod" = "$i" ] ; then + echo "$file" + break + fi + i=$((i+1)) + done +} + +script="${0##*/}" + +if [ "$1" = "-h" ] ; then + echo "$helpstr" | grep "$script:" | tr -s '\t' | cut -f 2- + exit +fi + +# Sometimes, UPATH ends up /dev/(null). +# That should be corrected, but for now... +# shellcheck disable=SC2154 +if [ ! -b "$VDEV_UPATH" ]; then + somepath="${VDEV_PATH}" +else + somepath="${VDEV_UPATH}" +fi + +if [ -b "$somepath" ] && PATH="/usr/sbin:$PATH" command -v smartctl > /dev/null || [ -n "$samples" ] ; then + if [ -n "$samples" ] ; then + # cat a smartctl output text file instead of running smartctl + # on a vdev (only used for developer testing). + file=$(get_filename_from_dir "$samples") + echo "file=$file" + raw_out=$(cat "$samples/$file") + else + raw_out=$(sudo smartctl -a "$somepath") + fi + + # What kind of drive are we? Look for the right line in smartctl: + # + # SAS: + # Transport protocol: SAS + # + # SATA: + # ATA Version is: 8 + # + # NVMe: + # SMART/Health Information (NVMe Log 0xnn, NSID 0xnn) + # + out=$(echo "$raw_out" | awk ' +# SAS specific +/read:/{print "rrd="$4"\nr_cor="$5"\nr_proc="$7"\nr_ucor="$8} +/write:/{print "rwr="$4"\nw_cor="$5"\nw_proc="$7"\nw_ucor="$8} +/Non-medium error count/{print "nonmed="$4} +/Elements in grown defect list/{print "defect="$6} + +# SAS common +/SAS/{type="sas"} +/Drive Temperature:/{print "temp="$4} +# Status can be a long string, substitute spaces for '_' +/SMART Health Status:/{printf "health="; for(i=4;i<=NF-1;i++){printf "%s_", $i}; printf "%s\n", $i} +/number of hours powered up/{print "hours_on="$7; hours_on=int($7)} +/Serial number:/{print "serial="$3} + +# SATA specific +/Reallocated_Sector_Ct/{print "realloc="$10} +/Reported_Uncorrect/{print "rep_ucor="$10} +/Command_Timeout/{print "cmd_to="$10} +/Current_Pending_Sector/{print "pend_sec="$10} +/Offline_Uncorrectable/{print "off_ucor="$10} +/ATA Error Count:/{print "ata_err="$4} +/Power_Cycle_Count/{print "pwr_cyc="$10} + +# SATA common +/SATA/{type="sata"} +/Temperature_Celsius/{print "temp="$10} +/Airflow_Temperature_Cel/{print "temp="$10} +/Current Temperature:/{print "temp="$3} +/SMART overall-health self-assessment test result:/{print "health="$6} +/Power_On_Hours/{print "hours_on="$10; hours_on=int($10)} +/Serial Number:/{print "serial="$3} + +# NVMe common +/NVMe/{type="nvme"} +/Temperature:/{print "temp="$2} +/SMART overall-health self-assessment test result:/{print "health="$6} +/Power On Hours:/{gsub("[^0-9]","",$4); print "hours_on="$4} +/Serial Number:/{print "serial="$3} +/Power Cycles:/{print "pwr_cyc="$3} + +# NVMe specific +/Media and Data Integrity Errors:/{print "nvme_err="$6} + +# SMART self-test info +/Self-test execution status:/{progress=tolower($4)} # SAS +/SMART Self-test log/{test_seen=1} # SAS +/SMART Extended Self-test Log/{test_seen=1} # SATA +/# 1/{ + test_type=tolower($3"_"$4); + # Status could be one word ("Completed") or multiple ("Completed: read + # failure"). Look for the ":" to see if we need to grab more words. + + if ($5 ~ ":") + status=tolower($5""$6"_"$7) + else + status=tolower($5) + if (status=="self") + status="running"; + + if (type == "sas") { + hours=int($(NF-4)) + } else { + hours=int($(NF-1)) + # SATA reports percent remaining, rather than percent done + # Convert it to percent done. + progress=(100-int($(NF-2)))"%" + } + # When we int()-ify "hours", it converts stuff like "NOW" and "-" into + # 0. In those cases, set it to hours_on, so they will cancel out in + # the "hours_ago" calculation later on. + if (hours == 0) + hours=hours_on + + if (test_seen) { + print "test="hours_on + print "test_type="test_type + print "test_status="status + print "test_progress="progress + } + # Not all drives report hours_on + if (hours_on && hours) { + total_hours_ago=(hours_on-hours) + days_ago=int(total_hours_ago/24) + hours_ago=(total_hours_ago % 24) + if (days_ago != 0) + ago_str=days_ago"d" + if (hours_ago !=0) + ago_str=ago_str""hours_ago"h" + print "test_ended="ago_str + } +} + +END {print "type="type; ORS="\n"; print ""} +'); +fi +type=$(echo "$out" | grep '^type=' | cut -d '=' -f 2) + +# If type is not set by now, either we don't have a block device +# or smartctl failed. Either way, default to ATA and set $out to +# nothing. +if [ -z "$type" ]; then + type="sata" + out= +fi + +case $script in +smart) + # Print temperature plus common predictors of drive failure + if [ "$type" = "sas" ] ; then + scripts="temp|health|r_ucor|w_ucor" + elif [ "$type" = "sata" ] ; then + scripts="temp|health|ata_err|realloc|rep_ucor|cmd_to|pend_sec|off_ucor" + elif [ "$type" = "nvme" ] ; then + scripts="temp|health|nvme_err" + fi + ;; +smartx) + # Print some other interesting stats + if [ "$type" = "sas" ] ; then + scripts="hours_on|defect|nonmed|r_proc|w_proc" + elif [ "$type" = "sata" ] ; then + scripts="hours_on|pwr_cyc" + elif [ "$type" = "nvme" ] ; then + scripts="hours_on|pwr_cyc" + fi + ;; +smart_test) + scripts="test_type|test_status|test_progress|test_ended" + ;; +*) + scripts="$script" +esac + +with_vals=$(echo "$out" | grep -E "$scripts") +if [ -n "$with_vals" ]; then + echo "$with_vals" + without_vals=$(echo "$scripts" | tr '|' '\n' | + grep -v -E "$(echo "$with_vals" | + awk -F "=" '{print $1}')" | awk '{print $0"="}') +else + without_vals=$(echo "$scripts" | tr '|' '\n' | awk '{print $0"="}') +fi + +if [ -n "$without_vals" ]; then + echo "$without_vals" +fi diff --git a/straper/db/public/zfs/etc-zfs/zpool.d/dm-deps b/straper/db/public/zfs/etc-zfs/zpool.d/dm-deps new file mode 100755 index 0000000..4422478 --- /dev/null +++ b/straper/db/public/zfs/etc-zfs/zpool.d/dm-deps @@ -0,0 +1,28 @@ +#!/bin/sh +# +# Show device mapper dependent / underlying devices. This is useful for +# looking up the /dev/sd* devices associated with a dm or multipath device. +# + +if [ "$1" = "-h" ] ; then + echo "Show device mapper dependent (underlying) devices." + exit +fi + +# shellcheck disable=SC2154 +dev="$VDEV_PATH" + +# If the VDEV path is a symlink, resolve it to a real device +if [ -L "$dev" ] ; then + dev=$(readlink "$dev") +fi + +dev="${dev##*/}" +val="" +if [ -d "/sys/class/block/$dev/slaves" ] ; then + # ls -C: output in columns, no newlines, two spaces (change to one) + # shellcheck disable=SC2012 + val=$(ls -C "/sys/class/block/$dev/slaves" | tr -s '[:space:]' ' ') +fi + +echo "dm-deps=$val" diff --git a/straper/db/public/zfs/etc-zfs/zpool.d/enc b/straper/db/public/zfs/etc-zfs/zpool.d/enc new file mode 100755 index 0000000..19ef92a --- /dev/null +++ b/straper/db/public/zfs/etc-zfs/zpool.d/enc @@ -0,0 +1,69 @@ +#!/bin/sh +# +# Print SCSI Enclosure Services (SES) info. The output is dependent on the name +# of the script/symlink used to call it. +# +helpstr=" +enc: Show disk enclosure w:x:y:z value. +slot: Show disk slot number as reported by the enclosure. +encdev: Show /dev/sg* device associated with the enclosure disk slot. +fault_led: Show value of the disk enclosure slot fault LED. +locate_led: Show value of the disk enclosure slot locate LED. +ses: Show disk's enc, enc device, slot, and fault/locate LED values." + +script="${0##*/}" +if [ "$1" = "-h" ] ; then + echo "$helpstr" | grep "$script:" | tr -s '\t' | cut -f 2- + exit +fi + +if [ "$script" = "ses" ] ; then + scripts='enc encdev slot fault_led locate_led' +else + scripts="$script" +fi + +for i in $scripts ; do + # shellcheck disable=SC2154 + if [ -z "$VDEV_ENC_SYSFS_PATH" ] ; then + echo "$i=" + continue + fi + + val="" + case $i in + enc) + if echo "$VDEV_ENC_SYSFS_PATH" | grep -q '/sys/bus/pci/slots' ; then + val="$VDEV_ENC_SYSFS_PATH" + else + val="$(ls """$VDEV_ENC_SYSFS_PATH/../../""" 2>/dev/null)" + fi + ;; + slot) + if echo "$VDEV_ENC_SYSFS_PATH" | grep -q '/sys/bus/pci/slots' ; then + val="$(basename """$VDEV_ENC_SYSFS_PATH""")" + else + val="$(cat """$VDEV_ENC_SYSFS_PATH/slot""" 2>/dev/null)" + fi + ;; + encdev) + val=$(ls "$VDEV_ENC_SYSFS_PATH/../device/scsi_generic" 2>/dev/null) + ;; + fault_led) + # JBODs fault LED is called 'fault', NVMe fault LED is called + # 'attention'. + if [ -f "$VDEV_ENC_SYSFS_PATH/fault" ] ; then + val=$(cat "$VDEV_ENC_SYSFS_PATH/fault" 2>/dev/null) + elif [ -f "$VDEV_ENC_SYSFS_PATH/attention" ] ; then + val=$(cat "$VDEV_ENC_SYSFS_PATH/attention" 2>/dev/null) + fi + ;; + locate_led) + val=$(cat "$VDEV_ENC_SYSFS_PATH/locate" 2>/dev/null) + ;; + *) + val=invalid + ;; + esac + echo "$i=$val" +done diff --git a/straper/db/public/zfs/etc-zfs/zpool.d/encdev b/straper/db/public/zfs/etc-zfs/zpool.d/encdev new file mode 100755 index 0000000..19ef92a --- /dev/null +++ b/straper/db/public/zfs/etc-zfs/zpool.d/encdev @@ -0,0 +1,69 @@ +#!/bin/sh +# +# Print SCSI Enclosure Services (SES) info. The output is dependent on the name +# of the script/symlink used to call it. +# +helpstr=" +enc: Show disk enclosure w:x:y:z value. +slot: Show disk slot number as reported by the enclosure. +encdev: Show /dev/sg* device associated with the enclosure disk slot. +fault_led: Show value of the disk enclosure slot fault LED. +locate_led: Show value of the disk enclosure slot locate LED. +ses: Show disk's enc, enc device, slot, and fault/locate LED values." + +script="${0##*/}" +if [ "$1" = "-h" ] ; then + echo "$helpstr" | grep "$script:" | tr -s '\t' | cut -f 2- + exit +fi + +if [ "$script" = "ses" ] ; then + scripts='enc encdev slot fault_led locate_led' +else + scripts="$script" +fi + +for i in $scripts ; do + # shellcheck disable=SC2154 + if [ -z "$VDEV_ENC_SYSFS_PATH" ] ; then + echo "$i=" + continue + fi + + val="" + case $i in + enc) + if echo "$VDEV_ENC_SYSFS_PATH" | grep -q '/sys/bus/pci/slots' ; then + val="$VDEV_ENC_SYSFS_PATH" + else + val="$(ls """$VDEV_ENC_SYSFS_PATH/../../""" 2>/dev/null)" + fi + ;; + slot) + if echo "$VDEV_ENC_SYSFS_PATH" | grep -q '/sys/bus/pci/slots' ; then + val="$(basename """$VDEV_ENC_SYSFS_PATH""")" + else + val="$(cat """$VDEV_ENC_SYSFS_PATH/slot""" 2>/dev/null)" + fi + ;; + encdev) + val=$(ls "$VDEV_ENC_SYSFS_PATH/../device/scsi_generic" 2>/dev/null) + ;; + fault_led) + # JBODs fault LED is called 'fault', NVMe fault LED is called + # 'attention'. + if [ -f "$VDEV_ENC_SYSFS_PATH/fault" ] ; then + val=$(cat "$VDEV_ENC_SYSFS_PATH/fault" 2>/dev/null) + elif [ -f "$VDEV_ENC_SYSFS_PATH/attention" ] ; then + val=$(cat "$VDEV_ENC_SYSFS_PATH/attention" 2>/dev/null) + fi + ;; + locate_led) + val=$(cat "$VDEV_ENC_SYSFS_PATH/locate" 2>/dev/null) + ;; + *) + val=invalid + ;; + esac + echo "$i=$val" +done diff --git a/straper/db/public/zfs/etc-zfs/zpool.d/fault_led b/straper/db/public/zfs/etc-zfs/zpool.d/fault_led new file mode 100755 index 0000000..19ef92a --- /dev/null +++ b/straper/db/public/zfs/etc-zfs/zpool.d/fault_led @@ -0,0 +1,69 @@ +#!/bin/sh +# +# Print SCSI Enclosure Services (SES) info. The output is dependent on the name +# of the script/symlink used to call it. +# +helpstr=" +enc: Show disk enclosure w:x:y:z value. +slot: Show disk slot number as reported by the enclosure. +encdev: Show /dev/sg* device associated with the enclosure disk slot. +fault_led: Show value of the disk enclosure slot fault LED. +locate_led: Show value of the disk enclosure slot locate LED. +ses: Show disk's enc, enc device, slot, and fault/locate LED values." + +script="${0##*/}" +if [ "$1" = "-h" ] ; then + echo "$helpstr" | grep "$script:" | tr -s '\t' | cut -f 2- + exit +fi + +if [ "$script" = "ses" ] ; then + scripts='enc encdev slot fault_led locate_led' +else + scripts="$script" +fi + +for i in $scripts ; do + # shellcheck disable=SC2154 + if [ -z "$VDEV_ENC_SYSFS_PATH" ] ; then + echo "$i=" + continue + fi + + val="" + case $i in + enc) + if echo "$VDEV_ENC_SYSFS_PATH" | grep -q '/sys/bus/pci/slots' ; then + val="$VDEV_ENC_SYSFS_PATH" + else + val="$(ls """$VDEV_ENC_SYSFS_PATH/../../""" 2>/dev/null)" + fi + ;; + slot) + if echo "$VDEV_ENC_SYSFS_PATH" | grep -q '/sys/bus/pci/slots' ; then + val="$(basename """$VDEV_ENC_SYSFS_PATH""")" + else + val="$(cat """$VDEV_ENC_SYSFS_PATH/slot""" 2>/dev/null)" + fi + ;; + encdev) + val=$(ls "$VDEV_ENC_SYSFS_PATH/../device/scsi_generic" 2>/dev/null) + ;; + fault_led) + # JBODs fault LED is called 'fault', NVMe fault LED is called + # 'attention'. + if [ -f "$VDEV_ENC_SYSFS_PATH/fault" ] ; then + val=$(cat "$VDEV_ENC_SYSFS_PATH/fault" 2>/dev/null) + elif [ -f "$VDEV_ENC_SYSFS_PATH/attention" ] ; then + val=$(cat "$VDEV_ENC_SYSFS_PATH/attention" 2>/dev/null) + fi + ;; + locate_led) + val=$(cat "$VDEV_ENC_SYSFS_PATH/locate" 2>/dev/null) + ;; + *) + val=invalid + ;; + esac + echo "$i=$val" +done diff --git a/straper/db/public/zfs/etc-zfs/zpool.d/health b/straper/db/public/zfs/etc-zfs/zpool.d/health new file mode 100755 index 0000000..8ad3e10 --- /dev/null +++ b/straper/db/public/zfs/etc-zfs/zpool.d/health @@ -0,0 +1,249 @@ +#!/bin/sh +# +# Show SMART stats +# + +helpstr=" +smart: Show SMART temperature and error stats (specific to drive type) +smartx: Show SMART extended drive stats (specific to drive type). +temp: Show SMART drive temperature in celsius (all drives). +health: Show reported SMART status (all drives). +r_proc: Show SMART read GBytes processed over drive lifetime (SAS). +w_proc: Show SMART write GBytes processed over drive lifetime (SAS). +r_ucor: Show SMART read uncorrectable errors (SAS). +w_ucor: Show SMART write uncorrectable errors (SAS). +nonmed: Show SMART non-medium errors (SAS). +defect: Show SMART grown defect list (SAS). +hours_on: Show number of hours drive powered on (all drives). +realloc: Show SMART reallocated sectors count (ATA). +rep_ucor: Show SMART reported uncorrectable count (ATA). +cmd_to: Show SMART command timeout count (ATA). +pend_sec: Show SMART current pending sector count (ATA). +off_ucor: Show SMART offline uncorrectable errors (ATA). +ata_err: Show SMART ATA errors (ATA). +pwr_cyc: Show SMART power cycle count (ATA). +serial: Show disk serial number. +nvme_err: Show SMART NVMe errors (NVMe). +smart_test: Show SMART self-test results summary. +test_type: Show SMART self-test type (short, long... ). +test_status: Show SMART self-test status. +test_progress: Show SMART self-test percentage done. +test_ended: Show when the last SMART self-test ended (if supported). +" + +# Hack for developer testing +# +# If you set $samples to a directory containing smartctl output text files, +# we will use them instead of running smartctl on the vdevs. This can be +# useful if you want to test a bunch of different smartctl outputs. Also, if +# $samples is set, and additional 'file' column is added to the zpool output +# showing the filename. +samples= + +# get_filename_from_dir DIR +# +# Look in directory DIR and return a filename from it. The filename returned +# is chosen quasi-sequentially (based off our PID). This allows us to return +# a different filename every time this script is invoked (which we do for each +# vdev), without having to maintain state. +get_filename_from_dir() +{ + dir=$1 + pid="$$" + num_files=$(find "$dir" -maxdepth 1 -type f | wc -l) + mod=$((pid % num_files)) + i=0 + find "$dir" -type f -printf '%f\n' | while read -r file ; do + if [ "$mod" = "$i" ] ; then + echo "$file" + break + fi + i=$((i+1)) + done +} + +script="${0##*/}" + +if [ "$1" = "-h" ] ; then + echo "$helpstr" | grep "$script:" | tr -s '\t' | cut -f 2- + exit +fi + +# Sometimes, UPATH ends up /dev/(null). +# That should be corrected, but for now... +# shellcheck disable=SC2154 +if [ ! -b "$VDEV_UPATH" ]; then + somepath="${VDEV_PATH}" +else + somepath="${VDEV_UPATH}" +fi + +if [ -b "$somepath" ] && PATH="/usr/sbin:$PATH" command -v smartctl > /dev/null || [ -n "$samples" ] ; then + if [ -n "$samples" ] ; then + # cat a smartctl output text file instead of running smartctl + # on a vdev (only used for developer testing). + file=$(get_filename_from_dir "$samples") + echo "file=$file" + raw_out=$(cat "$samples/$file") + else + raw_out=$(sudo smartctl -a "$somepath") + fi + + # What kind of drive are we? Look for the right line in smartctl: + # + # SAS: + # Transport protocol: SAS + # + # SATA: + # ATA Version is: 8 + # + # NVMe: + # SMART/Health Information (NVMe Log 0xnn, NSID 0xnn) + # + out=$(echo "$raw_out" | awk ' +# SAS specific +/read:/{print "rrd="$4"\nr_cor="$5"\nr_proc="$7"\nr_ucor="$8} +/write:/{print "rwr="$4"\nw_cor="$5"\nw_proc="$7"\nw_ucor="$8} +/Non-medium error count/{print "nonmed="$4} +/Elements in grown defect list/{print "defect="$6} + +# SAS common +/SAS/{type="sas"} +/Drive Temperature:/{print "temp="$4} +# Status can be a long string, substitute spaces for '_' +/SMART Health Status:/{printf "health="; for(i=4;i<=NF-1;i++){printf "%s_", $i}; printf "%s\n", $i} +/number of hours powered up/{print "hours_on="$7; hours_on=int($7)} +/Serial number:/{print "serial="$3} + +# SATA specific +/Reallocated_Sector_Ct/{print "realloc="$10} +/Reported_Uncorrect/{print "rep_ucor="$10} +/Command_Timeout/{print "cmd_to="$10} +/Current_Pending_Sector/{print "pend_sec="$10} +/Offline_Uncorrectable/{print "off_ucor="$10} +/ATA Error Count:/{print "ata_err="$4} +/Power_Cycle_Count/{print "pwr_cyc="$10} + +# SATA common +/SATA/{type="sata"} +/Temperature_Celsius/{print "temp="$10} +/Airflow_Temperature_Cel/{print "temp="$10} +/Current Temperature:/{print "temp="$3} +/SMART overall-health self-assessment test result:/{print "health="$6} +/Power_On_Hours/{print "hours_on="$10; hours_on=int($10)} +/Serial Number:/{print "serial="$3} + +# NVMe common +/NVMe/{type="nvme"} +/Temperature:/{print "temp="$2} +/SMART overall-health self-assessment test result:/{print "health="$6} +/Power On Hours:/{gsub("[^0-9]","",$4); print "hours_on="$4} +/Serial Number:/{print "serial="$3} +/Power Cycles:/{print "pwr_cyc="$3} + +# NVMe specific +/Media and Data Integrity Errors:/{print "nvme_err="$6} + +# SMART self-test info +/Self-test execution status:/{progress=tolower($4)} # SAS +/SMART Self-test log/{test_seen=1} # SAS +/SMART Extended Self-test Log/{test_seen=1} # SATA +/# 1/{ + test_type=tolower($3"_"$4); + # Status could be one word ("Completed") or multiple ("Completed: read + # failure"). Look for the ":" to see if we need to grab more words. + + if ($5 ~ ":") + status=tolower($5""$6"_"$7) + else + status=tolower($5) + if (status=="self") + status="running"; + + if (type == "sas") { + hours=int($(NF-4)) + } else { + hours=int($(NF-1)) + # SATA reports percent remaining, rather than percent done + # Convert it to percent done. + progress=(100-int($(NF-2)))"%" + } + # When we int()-ify "hours", it converts stuff like "NOW" and "-" into + # 0. In those cases, set it to hours_on, so they will cancel out in + # the "hours_ago" calculation later on. + if (hours == 0) + hours=hours_on + + if (test_seen) { + print "test="hours_on + print "test_type="test_type + print "test_status="status + print "test_progress="progress + } + # Not all drives report hours_on + if (hours_on && hours) { + total_hours_ago=(hours_on-hours) + days_ago=int(total_hours_ago/24) + hours_ago=(total_hours_ago % 24) + if (days_ago != 0) + ago_str=days_ago"d" + if (hours_ago !=0) + ago_str=ago_str""hours_ago"h" + print "test_ended="ago_str + } +} + +END {print "type="type; ORS="\n"; print ""} +'); +fi +type=$(echo "$out" | grep '^type=' | cut -d '=' -f 2) + +# If type is not set by now, either we don't have a block device +# or smartctl failed. Either way, default to ATA and set $out to +# nothing. +if [ -z "$type" ]; then + type="sata" + out= +fi + +case $script in +smart) + # Print temperature plus common predictors of drive failure + if [ "$type" = "sas" ] ; then + scripts="temp|health|r_ucor|w_ucor" + elif [ "$type" = "sata" ] ; then + scripts="temp|health|ata_err|realloc|rep_ucor|cmd_to|pend_sec|off_ucor" + elif [ "$type" = "nvme" ] ; then + scripts="temp|health|nvme_err" + fi + ;; +smartx) + # Print some other interesting stats + if [ "$type" = "sas" ] ; then + scripts="hours_on|defect|nonmed|r_proc|w_proc" + elif [ "$type" = "sata" ] ; then + scripts="hours_on|pwr_cyc" + elif [ "$type" = "nvme" ] ; then + scripts="hours_on|pwr_cyc" + fi + ;; +smart_test) + scripts="test_type|test_status|test_progress|test_ended" + ;; +*) + scripts="$script" +esac + +with_vals=$(echo "$out" | grep -E "$scripts") +if [ -n "$with_vals" ]; then + echo "$with_vals" + without_vals=$(echo "$scripts" | tr '|' '\n' | + grep -v -E "$(echo "$with_vals" | + awk -F "=" '{print $1}')" | awk '{print $0"="}') +else + without_vals=$(echo "$scripts" | tr '|' '\n' | awk '{print $0"="}') +fi + +if [ -n "$without_vals" ]; then + echo "$without_vals" +fi diff --git a/straper/db/public/zfs/etc-zfs/zpool.d/hours_on b/straper/db/public/zfs/etc-zfs/zpool.d/hours_on new file mode 100755 index 0000000..8ad3e10 --- /dev/null +++ b/straper/db/public/zfs/etc-zfs/zpool.d/hours_on @@ -0,0 +1,249 @@ +#!/bin/sh +# +# Show SMART stats +# + +helpstr=" +smart: Show SMART temperature and error stats (specific to drive type) +smartx: Show SMART extended drive stats (specific to drive type). +temp: Show SMART drive temperature in celsius (all drives). +health: Show reported SMART status (all drives). +r_proc: Show SMART read GBytes processed over drive lifetime (SAS). +w_proc: Show SMART write GBytes processed over drive lifetime (SAS). +r_ucor: Show SMART read uncorrectable errors (SAS). +w_ucor: Show SMART write uncorrectable errors (SAS). +nonmed: Show SMART non-medium errors (SAS). +defect: Show SMART grown defect list (SAS). +hours_on: Show number of hours drive powered on (all drives). +realloc: Show SMART reallocated sectors count (ATA). +rep_ucor: Show SMART reported uncorrectable count (ATA). +cmd_to: Show SMART command timeout count (ATA). +pend_sec: Show SMART current pending sector count (ATA). +off_ucor: Show SMART offline uncorrectable errors (ATA). +ata_err: Show SMART ATA errors (ATA). +pwr_cyc: Show SMART power cycle count (ATA). +serial: Show disk serial number. +nvme_err: Show SMART NVMe errors (NVMe). +smart_test: Show SMART self-test results summary. +test_type: Show SMART self-test type (short, long... ). +test_status: Show SMART self-test status. +test_progress: Show SMART self-test percentage done. +test_ended: Show when the last SMART self-test ended (if supported). +" + +# Hack for developer testing +# +# If you set $samples to a directory containing smartctl output text files, +# we will use them instead of running smartctl on the vdevs. This can be +# useful if you want to test a bunch of different smartctl outputs. Also, if +# $samples is set, and additional 'file' column is added to the zpool output +# showing the filename. +samples= + +# get_filename_from_dir DIR +# +# Look in directory DIR and return a filename from it. The filename returned +# is chosen quasi-sequentially (based off our PID). This allows us to return +# a different filename every time this script is invoked (which we do for each +# vdev), without having to maintain state. +get_filename_from_dir() +{ + dir=$1 + pid="$$" + num_files=$(find "$dir" -maxdepth 1 -type f | wc -l) + mod=$((pid % num_files)) + i=0 + find "$dir" -type f -printf '%f\n' | while read -r file ; do + if [ "$mod" = "$i" ] ; then + echo "$file" + break + fi + i=$((i+1)) + done +} + +script="${0##*/}" + +if [ "$1" = "-h" ] ; then + echo "$helpstr" | grep "$script:" | tr -s '\t' | cut -f 2- + exit +fi + +# Sometimes, UPATH ends up /dev/(null). +# That should be corrected, but for now... +# shellcheck disable=SC2154 +if [ ! -b "$VDEV_UPATH" ]; then + somepath="${VDEV_PATH}" +else + somepath="${VDEV_UPATH}" +fi + +if [ -b "$somepath" ] && PATH="/usr/sbin:$PATH" command -v smartctl > /dev/null || [ -n "$samples" ] ; then + if [ -n "$samples" ] ; then + # cat a smartctl output text file instead of running smartctl + # on a vdev (only used for developer testing). + file=$(get_filename_from_dir "$samples") + echo "file=$file" + raw_out=$(cat "$samples/$file") + else + raw_out=$(sudo smartctl -a "$somepath") + fi + + # What kind of drive are we? Look for the right line in smartctl: + # + # SAS: + # Transport protocol: SAS + # + # SATA: + # ATA Version is: 8 + # + # NVMe: + # SMART/Health Information (NVMe Log 0xnn, NSID 0xnn) + # + out=$(echo "$raw_out" | awk ' +# SAS specific +/read:/{print "rrd="$4"\nr_cor="$5"\nr_proc="$7"\nr_ucor="$8} +/write:/{print "rwr="$4"\nw_cor="$5"\nw_proc="$7"\nw_ucor="$8} +/Non-medium error count/{print "nonmed="$4} +/Elements in grown defect list/{print "defect="$6} + +# SAS common +/SAS/{type="sas"} +/Drive Temperature:/{print "temp="$4} +# Status can be a long string, substitute spaces for '_' +/SMART Health Status:/{printf "health="; for(i=4;i<=NF-1;i++){printf "%s_", $i}; printf "%s\n", $i} +/number of hours powered up/{print "hours_on="$7; hours_on=int($7)} +/Serial number:/{print "serial="$3} + +# SATA specific +/Reallocated_Sector_Ct/{print "realloc="$10} +/Reported_Uncorrect/{print "rep_ucor="$10} +/Command_Timeout/{print "cmd_to="$10} +/Current_Pending_Sector/{print "pend_sec="$10} +/Offline_Uncorrectable/{print "off_ucor="$10} +/ATA Error Count:/{print "ata_err="$4} +/Power_Cycle_Count/{print "pwr_cyc="$10} + +# SATA common +/SATA/{type="sata"} +/Temperature_Celsius/{print "temp="$10} +/Airflow_Temperature_Cel/{print "temp="$10} +/Current Temperature:/{print "temp="$3} +/SMART overall-health self-assessment test result:/{print "health="$6} +/Power_On_Hours/{print "hours_on="$10; hours_on=int($10)} +/Serial Number:/{print "serial="$3} + +# NVMe common +/NVMe/{type="nvme"} +/Temperature:/{print "temp="$2} +/SMART overall-health self-assessment test result:/{print "health="$6} +/Power On Hours:/{gsub("[^0-9]","",$4); print "hours_on="$4} +/Serial Number:/{print "serial="$3} +/Power Cycles:/{print "pwr_cyc="$3} + +# NVMe specific +/Media and Data Integrity Errors:/{print "nvme_err="$6} + +# SMART self-test info +/Self-test execution status:/{progress=tolower($4)} # SAS +/SMART Self-test log/{test_seen=1} # SAS +/SMART Extended Self-test Log/{test_seen=1} # SATA +/# 1/{ + test_type=tolower($3"_"$4); + # Status could be one word ("Completed") or multiple ("Completed: read + # failure"). Look for the ":" to see if we need to grab more words. + + if ($5 ~ ":") + status=tolower($5""$6"_"$7) + else + status=tolower($5) + if (status=="self") + status="running"; + + if (type == "sas") { + hours=int($(NF-4)) + } else { + hours=int($(NF-1)) + # SATA reports percent remaining, rather than percent done + # Convert it to percent done. + progress=(100-int($(NF-2)))"%" + } + # When we int()-ify "hours", it converts stuff like "NOW" and "-" into + # 0. In those cases, set it to hours_on, so they will cancel out in + # the "hours_ago" calculation later on. + if (hours == 0) + hours=hours_on + + if (test_seen) { + print "test="hours_on + print "test_type="test_type + print "test_status="status + print "test_progress="progress + } + # Not all drives report hours_on + if (hours_on && hours) { + total_hours_ago=(hours_on-hours) + days_ago=int(total_hours_ago/24) + hours_ago=(total_hours_ago % 24) + if (days_ago != 0) + ago_str=days_ago"d" + if (hours_ago !=0) + ago_str=ago_str""hours_ago"h" + print "test_ended="ago_str + } +} + +END {print "type="type; ORS="\n"; print ""} +'); +fi +type=$(echo "$out" | grep '^type=' | cut -d '=' -f 2) + +# If type is not set by now, either we don't have a block device +# or smartctl failed. Either way, default to ATA and set $out to +# nothing. +if [ -z "$type" ]; then + type="sata" + out= +fi + +case $script in +smart) + # Print temperature plus common predictors of drive failure + if [ "$type" = "sas" ] ; then + scripts="temp|health|r_ucor|w_ucor" + elif [ "$type" = "sata" ] ; then + scripts="temp|health|ata_err|realloc|rep_ucor|cmd_to|pend_sec|off_ucor" + elif [ "$type" = "nvme" ] ; then + scripts="temp|health|nvme_err" + fi + ;; +smartx) + # Print some other interesting stats + if [ "$type" = "sas" ] ; then + scripts="hours_on|defect|nonmed|r_proc|w_proc" + elif [ "$type" = "sata" ] ; then + scripts="hours_on|pwr_cyc" + elif [ "$type" = "nvme" ] ; then + scripts="hours_on|pwr_cyc" + fi + ;; +smart_test) + scripts="test_type|test_status|test_progress|test_ended" + ;; +*) + scripts="$script" +esac + +with_vals=$(echo "$out" | grep -E "$scripts") +if [ -n "$with_vals" ]; then + echo "$with_vals" + without_vals=$(echo "$scripts" | tr '|' '\n' | + grep -v -E "$(echo "$with_vals" | + awk -F "=" '{print $1}')" | awk '{print $0"="}') +else + without_vals=$(echo "$scripts" | tr '|' '\n' | awk '{print $0"="}') +fi + +if [ -n "$without_vals" ]; then + echo "$without_vals" +fi diff --git a/straper/db/public/zfs/etc-zfs/zpool.d/iostat b/straper/db/public/zfs/etc-zfs/zpool.d/iostat new file mode 100755 index 0000000..2f8d79a --- /dev/null +++ b/straper/db/public/zfs/etc-zfs/zpool.d/iostat @@ -0,0 +1,86 @@ +#!/bin/sh +# +# Display most relevant iostat bandwidth/latency numbers. The output is +# dependent on the name of the script/symlink used to call it. +# + +helpstr=" +iostat: Show iostat values since boot (summary page). +iostat-1s: Do a single 1-second iostat sample and show values. +iostat-10s: Do a single 10-second iostat sample and show values." + +script="${0##*/}" +if [ "$1" = "-h" ] ; then + echo "$helpstr" | grep "$script:" | tr -s '\t' | cut -f 2- + exit +fi + +# Sometimes, UPATH ends up /dev/(null). +# That should be corrected, but for now... +# shellcheck disable=SC2154 +if [ ! -b "$VDEV_UPATH" ]; then + somepath="${VDEV_PATH}" +else + somepath="${VDEV_UPATH}" +fi + +if [ "$script" = "iostat-1s" ] ; then + # Do a single one-second sample + interval=1 + # Don't show summary stats + brief="yes" +elif [ "$script" = "iostat-10s" ] ; then + # Do a single ten-second sample + interval=10 + # Don't show summary stats + brief="yes" +fi + +if [ -f "$somepath" ] ; then + # We're a file-based vdev, iostat doesn't work on us. Do nothing. + exit +fi + +if [ "$(uname)" = "FreeBSD" ]; then + out=$(iostat -dKx \ + ${interval:+"-w $interval"} \ + ${interval:+"-c 1"} \ + "$somepath" | tail -n 2) +else + out=$(iostat -kx \ + ${brief:+"-y"} \ + ${interval:+"$interval"} \ + ${interval:+"1"} \ + "$somepath" | grep -v '^$' | tail -n 2) +fi + + +# Sample output (we want the last two lines): +# +# Linux 2.6.32-642.13.1.el6.x86_64 (centos68) 03/09/2017 _x86_64_ (6 CPU) +# +# avg-cpu: %user %nice %system %iowait %steal %idle +# 0.00 0.00 0.00 0.00 0.00 100.00 +# +# Device: rrqm/s wrqm/s r/s w/s rkB/s wkB/s avgrq-sz avgqu-sz await r_await w_await svctm %util +# sdb 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 +# + +# Get the column names +cols=$(echo "$out" | head -n 1) + +# Get the values and tab separate them to make them cut-able. +vals=$(echo "$out" | tail -n 1 | tr -s '[:space:]' '\t') + +i=0 +for col in $cols ; do + i=$((i+1)) + # Skip the first column since it's just the device name + if [ "$i" -eq 1 ]; then + continue + fi + + # Get i'th value + val=$(echo "$vals" | cut -f "$i") + echo "$col=$val" +done diff --git a/straper/db/public/zfs/etc-zfs/zpool.d/iostat-10s b/straper/db/public/zfs/etc-zfs/zpool.d/iostat-10s new file mode 100755 index 0000000..2f8d79a --- /dev/null +++ b/straper/db/public/zfs/etc-zfs/zpool.d/iostat-10s @@ -0,0 +1,86 @@ +#!/bin/sh +# +# Display most relevant iostat bandwidth/latency numbers. The output is +# dependent on the name of the script/symlink used to call it. +# + +helpstr=" +iostat: Show iostat values since boot (summary page). +iostat-1s: Do a single 1-second iostat sample and show values. +iostat-10s: Do a single 10-second iostat sample and show values." + +script="${0##*/}" +if [ "$1" = "-h" ] ; then + echo "$helpstr" | grep "$script:" | tr -s '\t' | cut -f 2- + exit +fi + +# Sometimes, UPATH ends up /dev/(null). +# That should be corrected, but for now... +# shellcheck disable=SC2154 +if [ ! -b "$VDEV_UPATH" ]; then + somepath="${VDEV_PATH}" +else + somepath="${VDEV_UPATH}" +fi + +if [ "$script" = "iostat-1s" ] ; then + # Do a single one-second sample + interval=1 + # Don't show summary stats + brief="yes" +elif [ "$script" = "iostat-10s" ] ; then + # Do a single ten-second sample + interval=10 + # Don't show summary stats + brief="yes" +fi + +if [ -f "$somepath" ] ; then + # We're a file-based vdev, iostat doesn't work on us. Do nothing. + exit +fi + +if [ "$(uname)" = "FreeBSD" ]; then + out=$(iostat -dKx \ + ${interval:+"-w $interval"} \ + ${interval:+"-c 1"} \ + "$somepath" | tail -n 2) +else + out=$(iostat -kx \ + ${brief:+"-y"} \ + ${interval:+"$interval"} \ + ${interval:+"1"} \ + "$somepath" | grep -v '^$' | tail -n 2) +fi + + +# Sample output (we want the last two lines): +# +# Linux 2.6.32-642.13.1.el6.x86_64 (centos68) 03/09/2017 _x86_64_ (6 CPU) +# +# avg-cpu: %user %nice %system %iowait %steal %idle +# 0.00 0.00 0.00 0.00 0.00 100.00 +# +# Device: rrqm/s wrqm/s r/s w/s rkB/s wkB/s avgrq-sz avgqu-sz await r_await w_await svctm %util +# sdb 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 +# + +# Get the column names +cols=$(echo "$out" | head -n 1) + +# Get the values and tab separate them to make them cut-able. +vals=$(echo "$out" | tail -n 1 | tr -s '[:space:]' '\t') + +i=0 +for col in $cols ; do + i=$((i+1)) + # Skip the first column since it's just the device name + if [ "$i" -eq 1 ]; then + continue + fi + + # Get i'th value + val=$(echo "$vals" | cut -f "$i") + echo "$col=$val" +done diff --git a/straper/db/public/zfs/etc-zfs/zpool.d/iostat-1s b/straper/db/public/zfs/etc-zfs/zpool.d/iostat-1s new file mode 100755 index 0000000..2f8d79a --- /dev/null +++ b/straper/db/public/zfs/etc-zfs/zpool.d/iostat-1s @@ -0,0 +1,86 @@ +#!/bin/sh +# +# Display most relevant iostat bandwidth/latency numbers. The output is +# dependent on the name of the script/symlink used to call it. +# + +helpstr=" +iostat: Show iostat values since boot (summary page). +iostat-1s: Do a single 1-second iostat sample and show values. +iostat-10s: Do a single 10-second iostat sample and show values." + +script="${0##*/}" +if [ "$1" = "-h" ] ; then + echo "$helpstr" | grep "$script:" | tr -s '\t' | cut -f 2- + exit +fi + +# Sometimes, UPATH ends up /dev/(null). +# That should be corrected, but for now... +# shellcheck disable=SC2154 +if [ ! -b "$VDEV_UPATH" ]; then + somepath="${VDEV_PATH}" +else + somepath="${VDEV_UPATH}" +fi + +if [ "$script" = "iostat-1s" ] ; then + # Do a single one-second sample + interval=1 + # Don't show summary stats + brief="yes" +elif [ "$script" = "iostat-10s" ] ; then + # Do a single ten-second sample + interval=10 + # Don't show summary stats + brief="yes" +fi + +if [ -f "$somepath" ] ; then + # We're a file-based vdev, iostat doesn't work on us. Do nothing. + exit +fi + +if [ "$(uname)" = "FreeBSD" ]; then + out=$(iostat -dKx \ + ${interval:+"-w $interval"} \ + ${interval:+"-c 1"} \ + "$somepath" | tail -n 2) +else + out=$(iostat -kx \ + ${brief:+"-y"} \ + ${interval:+"$interval"} \ + ${interval:+"1"} \ + "$somepath" | grep -v '^$' | tail -n 2) +fi + + +# Sample output (we want the last two lines): +# +# Linux 2.6.32-642.13.1.el6.x86_64 (centos68) 03/09/2017 _x86_64_ (6 CPU) +# +# avg-cpu: %user %nice %system %iowait %steal %idle +# 0.00 0.00 0.00 0.00 0.00 100.00 +# +# Device: rrqm/s wrqm/s r/s w/s rkB/s wkB/s avgrq-sz avgqu-sz await r_await w_await svctm %util +# sdb 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 +# + +# Get the column names +cols=$(echo "$out" | head -n 1) + +# Get the values and tab separate them to make them cut-able. +vals=$(echo "$out" | tail -n 1 | tr -s '[:space:]' '\t') + +i=0 +for col in $cols ; do + i=$((i+1)) + # Skip the first column since it's just the device name + if [ "$i" -eq 1 ]; then + continue + fi + + # Get i'th value + val=$(echo "$vals" | cut -f "$i") + echo "$col=$val" +done diff --git a/straper/db/public/zfs/etc-zfs/zpool.d/label b/straper/db/public/zfs/etc-zfs/zpool.d/label new file mode 100755 index 0000000..293effd --- /dev/null +++ b/straper/db/public/zfs/etc-zfs/zpool.d/label @@ -0,0 +1,92 @@ +#!/bin/sh +# +# Print some common lsblk values +# +# Any (lowercased) name symlinked to the lsblk script will be passed to lsblk +# as one of its --output names. Here's a partial list of --output names +# from the lsblk binary: +# +# Available columns (for --output): +# NAME device name +# KNAME internal kernel device name +# MAJ:MIN major:minor device number +# FSTYPE filesystem type +# MOUNTPOINT where the device is mounted +# LABEL filesystem LABEL +# UUID filesystem UUID +# RA read-ahead of the device +# RO read-only device +# RM removable device +# MODEL device identifier +# SIZE size of the device +# STATE state of the device +# OWNER user name +# GROUP group name +# MODE device node permissions +# ALIGNMENT alignment offset +# MIN-IO minimum I/O size +# OPT-IO optimal I/O size +# PHY-SEC physical sector size +# LOG-SEC logical sector size +# ROTA rotational device +# SCHED I/O scheduler name +# RQ-SIZE request queue size +# TYPE device type +# DISC-ALN discard alignment offset +# DISC-GRAN discard granularity +# DISC-MAX discard max bytes +# DISC-ZERO discard zeroes data +# +# If the script is run as just 'lsblk' then print out disk size, vendor, +# and model number. + + +helpstr=" +label: Show filesystem label. +model: Show disk model number. +size: Show the disk capacity. +vendor: Show the disk vendor. +lsblk: Show the disk size, vendor, and model number." + +script="${0##*/}" + +if [ "$1" = "-h" ] ; then + echo "$helpstr" | grep "$script:" | tr -s '\t' | cut -f 2- + exit +fi + +if [ "$script" = "lsblk" ] ; then + list="size vendor model" +else + list=$(echo "$script" | tr '[:upper:]' '[:lower:]') +fi + +# Sometimes, UPATH ends up /dev/(null). +# That should be corrected, but for now... +# shellcheck disable=SC2154 +if [ ! -b "$VDEV_UPATH" ]; then + somepath="${VDEV_PATH}" +else + somepath="${VDEV_UPATH}" +fi + +# Older versions of lsblk don't support all these values (like SERIAL). +for i in $list ; do + + # Special case: Looking up the size of a file-based vdev can't + # be done with lsblk. + if [ "$i" = "size" ] && [ -f "$somepath" ] ; then + size=$(du -h --apparent-size "$somepath" | cut -f 1) + echo "size=$size" + continue + fi + + + val="" + if val=$(eval "lsblk -dl -n -o $i $somepath 2>/dev/null") ; then + # Remove leading/trailing whitespace from value + val=$(echo "$val" | sed -e 's/^[[:space:]]*//' \ + -e 's/[[:space:]]*$//') + fi + echo "$i=$val" +done diff --git a/straper/db/public/zfs/etc-zfs/zpool.d/locate_led b/straper/db/public/zfs/etc-zfs/zpool.d/locate_led new file mode 100755 index 0000000..19ef92a --- /dev/null +++ b/straper/db/public/zfs/etc-zfs/zpool.d/locate_led @@ -0,0 +1,69 @@ +#!/bin/sh +# +# Print SCSI Enclosure Services (SES) info. The output is dependent on the name +# of the script/symlink used to call it. +# +helpstr=" +enc: Show disk enclosure w:x:y:z value. +slot: Show disk slot number as reported by the enclosure. +encdev: Show /dev/sg* device associated with the enclosure disk slot. +fault_led: Show value of the disk enclosure slot fault LED. +locate_led: Show value of the disk enclosure slot locate LED. +ses: Show disk's enc, enc device, slot, and fault/locate LED values." + +script="${0##*/}" +if [ "$1" = "-h" ] ; then + echo "$helpstr" | grep "$script:" | tr -s '\t' | cut -f 2- + exit +fi + +if [ "$script" = "ses" ] ; then + scripts='enc encdev slot fault_led locate_led' +else + scripts="$script" +fi + +for i in $scripts ; do + # shellcheck disable=SC2154 + if [ -z "$VDEV_ENC_SYSFS_PATH" ] ; then + echo "$i=" + continue + fi + + val="" + case $i in + enc) + if echo "$VDEV_ENC_SYSFS_PATH" | grep -q '/sys/bus/pci/slots' ; then + val="$VDEV_ENC_SYSFS_PATH" + else + val="$(ls """$VDEV_ENC_SYSFS_PATH/../../""" 2>/dev/null)" + fi + ;; + slot) + if echo "$VDEV_ENC_SYSFS_PATH" | grep -q '/sys/bus/pci/slots' ; then + val="$(basename """$VDEV_ENC_SYSFS_PATH""")" + else + val="$(cat """$VDEV_ENC_SYSFS_PATH/slot""" 2>/dev/null)" + fi + ;; + encdev) + val=$(ls "$VDEV_ENC_SYSFS_PATH/../device/scsi_generic" 2>/dev/null) + ;; + fault_led) + # JBODs fault LED is called 'fault', NVMe fault LED is called + # 'attention'. + if [ -f "$VDEV_ENC_SYSFS_PATH/fault" ] ; then + val=$(cat "$VDEV_ENC_SYSFS_PATH/fault" 2>/dev/null) + elif [ -f "$VDEV_ENC_SYSFS_PATH/attention" ] ; then + val=$(cat "$VDEV_ENC_SYSFS_PATH/attention" 2>/dev/null) + fi + ;; + locate_led) + val=$(cat "$VDEV_ENC_SYSFS_PATH/locate" 2>/dev/null) + ;; + *) + val=invalid + ;; + esac + echo "$i=$val" +done diff --git a/straper/db/public/zfs/etc-zfs/zpool.d/lsblk b/straper/db/public/zfs/etc-zfs/zpool.d/lsblk new file mode 100755 index 0000000..293effd --- /dev/null +++ b/straper/db/public/zfs/etc-zfs/zpool.d/lsblk @@ -0,0 +1,92 @@ +#!/bin/sh +# +# Print some common lsblk values +# +# Any (lowercased) name symlinked to the lsblk script will be passed to lsblk +# as one of its --output names. Here's a partial list of --output names +# from the lsblk binary: +# +# Available columns (for --output): +# NAME device name +# KNAME internal kernel device name +# MAJ:MIN major:minor device number +# FSTYPE filesystem type +# MOUNTPOINT where the device is mounted +# LABEL filesystem LABEL +# UUID filesystem UUID +# RA read-ahead of the device +# RO read-only device +# RM removable device +# MODEL device identifier +# SIZE size of the device +# STATE state of the device +# OWNER user name +# GROUP group name +# MODE device node permissions +# ALIGNMENT alignment offset +# MIN-IO minimum I/O size +# OPT-IO optimal I/O size +# PHY-SEC physical sector size +# LOG-SEC logical sector size +# ROTA rotational device +# SCHED I/O scheduler name +# RQ-SIZE request queue size +# TYPE device type +# DISC-ALN discard alignment offset +# DISC-GRAN discard granularity +# DISC-MAX discard max bytes +# DISC-ZERO discard zeroes data +# +# If the script is run as just 'lsblk' then print out disk size, vendor, +# and model number. + + +helpstr=" +label: Show filesystem label. +model: Show disk model number. +size: Show the disk capacity. +vendor: Show the disk vendor. +lsblk: Show the disk size, vendor, and model number." + +script="${0##*/}" + +if [ "$1" = "-h" ] ; then + echo "$helpstr" | grep "$script:" | tr -s '\t' | cut -f 2- + exit +fi + +if [ "$script" = "lsblk" ] ; then + list="size vendor model" +else + list=$(echo "$script" | tr '[:upper:]' '[:lower:]') +fi + +# Sometimes, UPATH ends up /dev/(null). +# That should be corrected, but for now... +# shellcheck disable=SC2154 +if [ ! -b "$VDEV_UPATH" ]; then + somepath="${VDEV_PATH}" +else + somepath="${VDEV_UPATH}" +fi + +# Older versions of lsblk don't support all these values (like SERIAL). +for i in $list ; do + + # Special case: Looking up the size of a file-based vdev can't + # be done with lsblk. + if [ "$i" = "size" ] && [ -f "$somepath" ] ; then + size=$(du -h --apparent-size "$somepath" | cut -f 1) + echo "size=$size" + continue + fi + + + val="" + if val=$(eval "lsblk -dl -n -o $i $somepath 2>/dev/null") ; then + # Remove leading/trailing whitespace from value + val=$(echo "$val" | sed -e 's/^[[:space:]]*//' \ + -e 's/[[:space:]]*$//') + fi + echo "$i=$val" +done diff --git a/straper/db/public/zfs/etc-zfs/zpool.d/media b/straper/db/public/zfs/etc-zfs/zpool.d/media new file mode 100755 index 0000000..095ac86 --- /dev/null +++ b/straper/db/public/zfs/etc-zfs/zpool.d/media @@ -0,0 +1,33 @@ +#!/bin/sh +# +# Print out the type of device +# + +if [ "$1" = "-h" ] ; then + echo "Show whether a vdev is a file, hdd, ssd, or iscsi." + exit +fi + +# shellcheck disable=SC2154 +if [ -b "$VDEV_UPATH" ]; then + device="${VDEV_UPATH##*/}" + read -r val 2>/dev/null < "/sys/block/$device/queue/rotational" + case "$val" in + 0) MEDIA="ssd" ;; + 1) MEDIA="hdd" ;; + *) MEDIA="invalid" ;; + esac + + vpd_pg83="/sys/block/$device/device/vpd_pg83" + if [ -f "$vpd_pg83" ]; then + if grep -q --binary "iqn." "$vpd_pg83"; then + MEDIA="iscsi" + fi + fi +else + if [ -f "$VDEV_UPATH" ]; then + MEDIA="file" + fi +fi + +echo "media=$MEDIA" diff --git a/straper/db/public/zfs/etc-zfs/zpool.d/model b/straper/db/public/zfs/etc-zfs/zpool.d/model new file mode 100755 index 0000000..293effd --- /dev/null +++ b/straper/db/public/zfs/etc-zfs/zpool.d/model @@ -0,0 +1,92 @@ +#!/bin/sh +# +# Print some common lsblk values +# +# Any (lowercased) name symlinked to the lsblk script will be passed to lsblk +# as one of its --output names. Here's a partial list of --output names +# from the lsblk binary: +# +# Available columns (for --output): +# NAME device name +# KNAME internal kernel device name +# MAJ:MIN major:minor device number +# FSTYPE filesystem type +# MOUNTPOINT where the device is mounted +# LABEL filesystem LABEL +# UUID filesystem UUID +# RA read-ahead of the device +# RO read-only device +# RM removable device +# MODEL device identifier +# SIZE size of the device +# STATE state of the device +# OWNER user name +# GROUP group name +# MODE device node permissions +# ALIGNMENT alignment offset +# MIN-IO minimum I/O size +# OPT-IO optimal I/O size +# PHY-SEC physical sector size +# LOG-SEC logical sector size +# ROTA rotational device +# SCHED I/O scheduler name +# RQ-SIZE request queue size +# TYPE device type +# DISC-ALN discard alignment offset +# DISC-GRAN discard granularity +# DISC-MAX discard max bytes +# DISC-ZERO discard zeroes data +# +# If the script is run as just 'lsblk' then print out disk size, vendor, +# and model number. + + +helpstr=" +label: Show filesystem label. +model: Show disk model number. +size: Show the disk capacity. +vendor: Show the disk vendor. +lsblk: Show the disk size, vendor, and model number." + +script="${0##*/}" + +if [ "$1" = "-h" ] ; then + echo "$helpstr" | grep "$script:" | tr -s '\t' | cut -f 2- + exit +fi + +if [ "$script" = "lsblk" ] ; then + list="size vendor model" +else + list=$(echo "$script" | tr '[:upper:]' '[:lower:]') +fi + +# Sometimes, UPATH ends up /dev/(null). +# That should be corrected, but for now... +# shellcheck disable=SC2154 +if [ ! -b "$VDEV_UPATH" ]; then + somepath="${VDEV_PATH}" +else + somepath="${VDEV_UPATH}" +fi + +# Older versions of lsblk don't support all these values (like SERIAL). +for i in $list ; do + + # Special case: Looking up the size of a file-based vdev can't + # be done with lsblk. + if [ "$i" = "size" ] && [ -f "$somepath" ] ; then + size=$(du -h --apparent-size "$somepath" | cut -f 1) + echo "size=$size" + continue + fi + + + val="" + if val=$(eval "lsblk -dl -n -o $i $somepath 2>/dev/null") ; then + # Remove leading/trailing whitespace from value + val=$(echo "$val" | sed -e 's/^[[:space:]]*//' \ + -e 's/[[:space:]]*$//') + fi + echo "$i=$val" +done diff --git a/straper/db/public/zfs/etc-zfs/zpool.d/nonmed b/straper/db/public/zfs/etc-zfs/zpool.d/nonmed new file mode 100755 index 0000000..8ad3e10 --- /dev/null +++ b/straper/db/public/zfs/etc-zfs/zpool.d/nonmed @@ -0,0 +1,249 @@ +#!/bin/sh +# +# Show SMART stats +# + +helpstr=" +smart: Show SMART temperature and error stats (specific to drive type) +smartx: Show SMART extended drive stats (specific to drive type). +temp: Show SMART drive temperature in celsius (all drives). +health: Show reported SMART status (all drives). +r_proc: Show SMART read GBytes processed over drive lifetime (SAS). +w_proc: Show SMART write GBytes processed over drive lifetime (SAS). +r_ucor: Show SMART read uncorrectable errors (SAS). +w_ucor: Show SMART write uncorrectable errors (SAS). +nonmed: Show SMART non-medium errors (SAS). +defect: Show SMART grown defect list (SAS). +hours_on: Show number of hours drive powered on (all drives). +realloc: Show SMART reallocated sectors count (ATA). +rep_ucor: Show SMART reported uncorrectable count (ATA). +cmd_to: Show SMART command timeout count (ATA). +pend_sec: Show SMART current pending sector count (ATA). +off_ucor: Show SMART offline uncorrectable errors (ATA). +ata_err: Show SMART ATA errors (ATA). +pwr_cyc: Show SMART power cycle count (ATA). +serial: Show disk serial number. +nvme_err: Show SMART NVMe errors (NVMe). +smart_test: Show SMART self-test results summary. +test_type: Show SMART self-test type (short, long... ). +test_status: Show SMART self-test status. +test_progress: Show SMART self-test percentage done. +test_ended: Show when the last SMART self-test ended (if supported). +" + +# Hack for developer testing +# +# If you set $samples to a directory containing smartctl output text files, +# we will use them instead of running smartctl on the vdevs. This can be +# useful if you want to test a bunch of different smartctl outputs. Also, if +# $samples is set, and additional 'file' column is added to the zpool output +# showing the filename. +samples= + +# get_filename_from_dir DIR +# +# Look in directory DIR and return a filename from it. The filename returned +# is chosen quasi-sequentially (based off our PID). This allows us to return +# a different filename every time this script is invoked (which we do for each +# vdev), without having to maintain state. +get_filename_from_dir() +{ + dir=$1 + pid="$$" + num_files=$(find "$dir" -maxdepth 1 -type f | wc -l) + mod=$((pid % num_files)) + i=0 + find "$dir" -type f -printf '%f\n' | while read -r file ; do + if [ "$mod" = "$i" ] ; then + echo "$file" + break + fi + i=$((i+1)) + done +} + +script="${0##*/}" + +if [ "$1" = "-h" ] ; then + echo "$helpstr" | grep "$script:" | tr -s '\t' | cut -f 2- + exit +fi + +# Sometimes, UPATH ends up /dev/(null). +# That should be corrected, but for now... +# shellcheck disable=SC2154 +if [ ! -b "$VDEV_UPATH" ]; then + somepath="${VDEV_PATH}" +else + somepath="${VDEV_UPATH}" +fi + +if [ -b "$somepath" ] && PATH="/usr/sbin:$PATH" command -v smartctl > /dev/null || [ -n "$samples" ] ; then + if [ -n "$samples" ] ; then + # cat a smartctl output text file instead of running smartctl + # on a vdev (only used for developer testing). + file=$(get_filename_from_dir "$samples") + echo "file=$file" + raw_out=$(cat "$samples/$file") + else + raw_out=$(sudo smartctl -a "$somepath") + fi + + # What kind of drive are we? Look for the right line in smartctl: + # + # SAS: + # Transport protocol: SAS + # + # SATA: + # ATA Version is: 8 + # + # NVMe: + # SMART/Health Information (NVMe Log 0xnn, NSID 0xnn) + # + out=$(echo "$raw_out" | awk ' +# SAS specific +/read:/{print "rrd="$4"\nr_cor="$5"\nr_proc="$7"\nr_ucor="$8} +/write:/{print "rwr="$4"\nw_cor="$5"\nw_proc="$7"\nw_ucor="$8} +/Non-medium error count/{print "nonmed="$4} +/Elements in grown defect list/{print "defect="$6} + +# SAS common +/SAS/{type="sas"} +/Drive Temperature:/{print "temp="$4} +# Status can be a long string, substitute spaces for '_' +/SMART Health Status:/{printf "health="; for(i=4;i<=NF-1;i++){printf "%s_", $i}; printf "%s\n", $i} +/number of hours powered up/{print "hours_on="$7; hours_on=int($7)} +/Serial number:/{print "serial="$3} + +# SATA specific +/Reallocated_Sector_Ct/{print "realloc="$10} +/Reported_Uncorrect/{print "rep_ucor="$10} +/Command_Timeout/{print "cmd_to="$10} +/Current_Pending_Sector/{print "pend_sec="$10} +/Offline_Uncorrectable/{print "off_ucor="$10} +/ATA Error Count:/{print "ata_err="$4} +/Power_Cycle_Count/{print "pwr_cyc="$10} + +# SATA common +/SATA/{type="sata"} +/Temperature_Celsius/{print "temp="$10} +/Airflow_Temperature_Cel/{print "temp="$10} +/Current Temperature:/{print "temp="$3} +/SMART overall-health self-assessment test result:/{print "health="$6} +/Power_On_Hours/{print "hours_on="$10; hours_on=int($10)} +/Serial Number:/{print "serial="$3} + +# NVMe common +/NVMe/{type="nvme"} +/Temperature:/{print "temp="$2} +/SMART overall-health self-assessment test result:/{print "health="$6} +/Power On Hours:/{gsub("[^0-9]","",$4); print "hours_on="$4} +/Serial Number:/{print "serial="$3} +/Power Cycles:/{print "pwr_cyc="$3} + +# NVMe specific +/Media and Data Integrity Errors:/{print "nvme_err="$6} + +# SMART self-test info +/Self-test execution status:/{progress=tolower($4)} # SAS +/SMART Self-test log/{test_seen=1} # SAS +/SMART Extended Self-test Log/{test_seen=1} # SATA +/# 1/{ + test_type=tolower($3"_"$4); + # Status could be one word ("Completed") or multiple ("Completed: read + # failure"). Look for the ":" to see if we need to grab more words. + + if ($5 ~ ":") + status=tolower($5""$6"_"$7) + else + status=tolower($5) + if (status=="self") + status="running"; + + if (type == "sas") { + hours=int($(NF-4)) + } else { + hours=int($(NF-1)) + # SATA reports percent remaining, rather than percent done + # Convert it to percent done. + progress=(100-int($(NF-2)))"%" + } + # When we int()-ify "hours", it converts stuff like "NOW" and "-" into + # 0. In those cases, set it to hours_on, so they will cancel out in + # the "hours_ago" calculation later on. + if (hours == 0) + hours=hours_on + + if (test_seen) { + print "test="hours_on + print "test_type="test_type + print "test_status="status + print "test_progress="progress + } + # Not all drives report hours_on + if (hours_on && hours) { + total_hours_ago=(hours_on-hours) + days_ago=int(total_hours_ago/24) + hours_ago=(total_hours_ago % 24) + if (days_ago != 0) + ago_str=days_ago"d" + if (hours_ago !=0) + ago_str=ago_str""hours_ago"h" + print "test_ended="ago_str + } +} + +END {print "type="type; ORS="\n"; print ""} +'); +fi +type=$(echo "$out" | grep '^type=' | cut -d '=' -f 2) + +# If type is not set by now, either we don't have a block device +# or smartctl failed. Either way, default to ATA and set $out to +# nothing. +if [ -z "$type" ]; then + type="sata" + out= +fi + +case $script in +smart) + # Print temperature plus common predictors of drive failure + if [ "$type" = "sas" ] ; then + scripts="temp|health|r_ucor|w_ucor" + elif [ "$type" = "sata" ] ; then + scripts="temp|health|ata_err|realloc|rep_ucor|cmd_to|pend_sec|off_ucor" + elif [ "$type" = "nvme" ] ; then + scripts="temp|health|nvme_err" + fi + ;; +smartx) + # Print some other interesting stats + if [ "$type" = "sas" ] ; then + scripts="hours_on|defect|nonmed|r_proc|w_proc" + elif [ "$type" = "sata" ] ; then + scripts="hours_on|pwr_cyc" + elif [ "$type" = "nvme" ] ; then + scripts="hours_on|pwr_cyc" + fi + ;; +smart_test) + scripts="test_type|test_status|test_progress|test_ended" + ;; +*) + scripts="$script" +esac + +with_vals=$(echo "$out" | grep -E "$scripts") +if [ -n "$with_vals" ]; then + echo "$with_vals" + without_vals=$(echo "$scripts" | tr '|' '\n' | + grep -v -E "$(echo "$with_vals" | + awk -F "=" '{print $1}')" | awk '{print $0"="}') +else + without_vals=$(echo "$scripts" | tr '|' '\n' | awk '{print $0"="}') +fi + +if [ -n "$without_vals" ]; then + echo "$without_vals" +fi diff --git a/straper/db/public/zfs/etc-zfs/zpool.d/nvme_err b/straper/db/public/zfs/etc-zfs/zpool.d/nvme_err new file mode 100755 index 0000000..8ad3e10 --- /dev/null +++ b/straper/db/public/zfs/etc-zfs/zpool.d/nvme_err @@ -0,0 +1,249 @@ +#!/bin/sh +# +# Show SMART stats +# + +helpstr=" +smart: Show SMART temperature and error stats (specific to drive type) +smartx: Show SMART extended drive stats (specific to drive type). +temp: Show SMART drive temperature in celsius (all drives). +health: Show reported SMART status (all drives). +r_proc: Show SMART read GBytes processed over drive lifetime (SAS). +w_proc: Show SMART write GBytes processed over drive lifetime (SAS). +r_ucor: Show SMART read uncorrectable errors (SAS). +w_ucor: Show SMART write uncorrectable errors (SAS). +nonmed: Show SMART non-medium errors (SAS). +defect: Show SMART grown defect list (SAS). +hours_on: Show number of hours drive powered on (all drives). +realloc: Show SMART reallocated sectors count (ATA). +rep_ucor: Show SMART reported uncorrectable count (ATA). +cmd_to: Show SMART command timeout count (ATA). +pend_sec: Show SMART current pending sector count (ATA). +off_ucor: Show SMART offline uncorrectable errors (ATA). +ata_err: Show SMART ATA errors (ATA). +pwr_cyc: Show SMART power cycle count (ATA). +serial: Show disk serial number. +nvme_err: Show SMART NVMe errors (NVMe). +smart_test: Show SMART self-test results summary. +test_type: Show SMART self-test type (short, long... ). +test_status: Show SMART self-test status. +test_progress: Show SMART self-test percentage done. +test_ended: Show when the last SMART self-test ended (if supported). +" + +# Hack for developer testing +# +# If you set $samples to a directory containing smartctl output text files, +# we will use them instead of running smartctl on the vdevs. This can be +# useful if you want to test a bunch of different smartctl outputs. Also, if +# $samples is set, and additional 'file' column is added to the zpool output +# showing the filename. +samples= + +# get_filename_from_dir DIR +# +# Look in directory DIR and return a filename from it. The filename returned +# is chosen quasi-sequentially (based off our PID). This allows us to return +# a different filename every time this script is invoked (which we do for each +# vdev), without having to maintain state. +get_filename_from_dir() +{ + dir=$1 + pid="$$" + num_files=$(find "$dir" -maxdepth 1 -type f | wc -l) + mod=$((pid % num_files)) + i=0 + find "$dir" -type f -printf '%f\n' | while read -r file ; do + if [ "$mod" = "$i" ] ; then + echo "$file" + break + fi + i=$((i+1)) + done +} + +script="${0##*/}" + +if [ "$1" = "-h" ] ; then + echo "$helpstr" | grep "$script:" | tr -s '\t' | cut -f 2- + exit +fi + +# Sometimes, UPATH ends up /dev/(null). +# That should be corrected, but for now... +# shellcheck disable=SC2154 +if [ ! -b "$VDEV_UPATH" ]; then + somepath="${VDEV_PATH}" +else + somepath="${VDEV_UPATH}" +fi + +if [ -b "$somepath" ] && PATH="/usr/sbin:$PATH" command -v smartctl > /dev/null || [ -n "$samples" ] ; then + if [ -n "$samples" ] ; then + # cat a smartctl output text file instead of running smartctl + # on a vdev (only used for developer testing). + file=$(get_filename_from_dir "$samples") + echo "file=$file" + raw_out=$(cat "$samples/$file") + else + raw_out=$(sudo smartctl -a "$somepath") + fi + + # What kind of drive are we? Look for the right line in smartctl: + # + # SAS: + # Transport protocol: SAS + # + # SATA: + # ATA Version is: 8 + # + # NVMe: + # SMART/Health Information (NVMe Log 0xnn, NSID 0xnn) + # + out=$(echo "$raw_out" | awk ' +# SAS specific +/read:/{print "rrd="$4"\nr_cor="$5"\nr_proc="$7"\nr_ucor="$8} +/write:/{print "rwr="$4"\nw_cor="$5"\nw_proc="$7"\nw_ucor="$8} +/Non-medium error count/{print "nonmed="$4} +/Elements in grown defect list/{print "defect="$6} + +# SAS common +/SAS/{type="sas"} +/Drive Temperature:/{print "temp="$4} +# Status can be a long string, substitute spaces for '_' +/SMART Health Status:/{printf "health="; for(i=4;i<=NF-1;i++){printf "%s_", $i}; printf "%s\n", $i} +/number of hours powered up/{print "hours_on="$7; hours_on=int($7)} +/Serial number:/{print "serial="$3} + +# SATA specific +/Reallocated_Sector_Ct/{print "realloc="$10} +/Reported_Uncorrect/{print "rep_ucor="$10} +/Command_Timeout/{print "cmd_to="$10} +/Current_Pending_Sector/{print "pend_sec="$10} +/Offline_Uncorrectable/{print "off_ucor="$10} +/ATA Error Count:/{print "ata_err="$4} +/Power_Cycle_Count/{print "pwr_cyc="$10} + +# SATA common +/SATA/{type="sata"} +/Temperature_Celsius/{print "temp="$10} +/Airflow_Temperature_Cel/{print "temp="$10} +/Current Temperature:/{print "temp="$3} +/SMART overall-health self-assessment test result:/{print "health="$6} +/Power_On_Hours/{print "hours_on="$10; hours_on=int($10)} +/Serial Number:/{print "serial="$3} + +# NVMe common +/NVMe/{type="nvme"} +/Temperature:/{print "temp="$2} +/SMART overall-health self-assessment test result:/{print "health="$6} +/Power On Hours:/{gsub("[^0-9]","",$4); print "hours_on="$4} +/Serial Number:/{print "serial="$3} +/Power Cycles:/{print "pwr_cyc="$3} + +# NVMe specific +/Media and Data Integrity Errors:/{print "nvme_err="$6} + +# SMART self-test info +/Self-test execution status:/{progress=tolower($4)} # SAS +/SMART Self-test log/{test_seen=1} # SAS +/SMART Extended Self-test Log/{test_seen=1} # SATA +/# 1/{ + test_type=tolower($3"_"$4); + # Status could be one word ("Completed") or multiple ("Completed: read + # failure"). Look for the ":" to see if we need to grab more words. + + if ($5 ~ ":") + status=tolower($5""$6"_"$7) + else + status=tolower($5) + if (status=="self") + status="running"; + + if (type == "sas") { + hours=int($(NF-4)) + } else { + hours=int($(NF-1)) + # SATA reports percent remaining, rather than percent done + # Convert it to percent done. + progress=(100-int($(NF-2)))"%" + } + # When we int()-ify "hours", it converts stuff like "NOW" and "-" into + # 0. In those cases, set it to hours_on, so they will cancel out in + # the "hours_ago" calculation later on. + if (hours == 0) + hours=hours_on + + if (test_seen) { + print "test="hours_on + print "test_type="test_type + print "test_status="status + print "test_progress="progress + } + # Not all drives report hours_on + if (hours_on && hours) { + total_hours_ago=(hours_on-hours) + days_ago=int(total_hours_ago/24) + hours_ago=(total_hours_ago % 24) + if (days_ago != 0) + ago_str=days_ago"d" + if (hours_ago !=0) + ago_str=ago_str""hours_ago"h" + print "test_ended="ago_str + } +} + +END {print "type="type; ORS="\n"; print ""} +'); +fi +type=$(echo "$out" | grep '^type=' | cut -d '=' -f 2) + +# If type is not set by now, either we don't have a block device +# or smartctl failed. Either way, default to ATA and set $out to +# nothing. +if [ -z "$type" ]; then + type="sata" + out= +fi + +case $script in +smart) + # Print temperature plus common predictors of drive failure + if [ "$type" = "sas" ] ; then + scripts="temp|health|r_ucor|w_ucor" + elif [ "$type" = "sata" ] ; then + scripts="temp|health|ata_err|realloc|rep_ucor|cmd_to|pend_sec|off_ucor" + elif [ "$type" = "nvme" ] ; then + scripts="temp|health|nvme_err" + fi + ;; +smartx) + # Print some other interesting stats + if [ "$type" = "sas" ] ; then + scripts="hours_on|defect|nonmed|r_proc|w_proc" + elif [ "$type" = "sata" ] ; then + scripts="hours_on|pwr_cyc" + elif [ "$type" = "nvme" ] ; then + scripts="hours_on|pwr_cyc" + fi + ;; +smart_test) + scripts="test_type|test_status|test_progress|test_ended" + ;; +*) + scripts="$script" +esac + +with_vals=$(echo "$out" | grep -E "$scripts") +if [ -n "$with_vals" ]; then + echo "$with_vals" + without_vals=$(echo "$scripts" | tr '|' '\n' | + grep -v -E "$(echo "$with_vals" | + awk -F "=" '{print $1}')" | awk '{print $0"="}') +else + without_vals=$(echo "$scripts" | tr '|' '\n' | awk '{print $0"="}') +fi + +if [ -n "$without_vals" ]; then + echo "$without_vals" +fi diff --git a/straper/db/public/zfs/etc-zfs/zpool.d/off_ucor b/straper/db/public/zfs/etc-zfs/zpool.d/off_ucor new file mode 100755 index 0000000..8ad3e10 --- /dev/null +++ b/straper/db/public/zfs/etc-zfs/zpool.d/off_ucor @@ -0,0 +1,249 @@ +#!/bin/sh +# +# Show SMART stats +# + +helpstr=" +smart: Show SMART temperature and error stats (specific to drive type) +smartx: Show SMART extended drive stats (specific to drive type). +temp: Show SMART drive temperature in celsius (all drives). +health: Show reported SMART status (all drives). +r_proc: Show SMART read GBytes processed over drive lifetime (SAS). +w_proc: Show SMART write GBytes processed over drive lifetime (SAS). +r_ucor: Show SMART read uncorrectable errors (SAS). +w_ucor: Show SMART write uncorrectable errors (SAS). +nonmed: Show SMART non-medium errors (SAS). +defect: Show SMART grown defect list (SAS). +hours_on: Show number of hours drive powered on (all drives). +realloc: Show SMART reallocated sectors count (ATA). +rep_ucor: Show SMART reported uncorrectable count (ATA). +cmd_to: Show SMART command timeout count (ATA). +pend_sec: Show SMART current pending sector count (ATA). +off_ucor: Show SMART offline uncorrectable errors (ATA). +ata_err: Show SMART ATA errors (ATA). +pwr_cyc: Show SMART power cycle count (ATA). +serial: Show disk serial number. +nvme_err: Show SMART NVMe errors (NVMe). +smart_test: Show SMART self-test results summary. +test_type: Show SMART self-test type (short, long... ). +test_status: Show SMART self-test status. +test_progress: Show SMART self-test percentage done. +test_ended: Show when the last SMART self-test ended (if supported). +" + +# Hack for developer testing +# +# If you set $samples to a directory containing smartctl output text files, +# we will use them instead of running smartctl on the vdevs. This can be +# useful if you want to test a bunch of different smartctl outputs. Also, if +# $samples is set, and additional 'file' column is added to the zpool output +# showing the filename. +samples= + +# get_filename_from_dir DIR +# +# Look in directory DIR and return a filename from it. The filename returned +# is chosen quasi-sequentially (based off our PID). This allows us to return +# a different filename every time this script is invoked (which we do for each +# vdev), without having to maintain state. +get_filename_from_dir() +{ + dir=$1 + pid="$$" + num_files=$(find "$dir" -maxdepth 1 -type f | wc -l) + mod=$((pid % num_files)) + i=0 + find "$dir" -type f -printf '%f\n' | while read -r file ; do + if [ "$mod" = "$i" ] ; then + echo "$file" + break + fi + i=$((i+1)) + done +} + +script="${0##*/}" + +if [ "$1" = "-h" ] ; then + echo "$helpstr" | grep "$script:" | tr -s '\t' | cut -f 2- + exit +fi + +# Sometimes, UPATH ends up /dev/(null). +# That should be corrected, but for now... +# shellcheck disable=SC2154 +if [ ! -b "$VDEV_UPATH" ]; then + somepath="${VDEV_PATH}" +else + somepath="${VDEV_UPATH}" +fi + +if [ -b "$somepath" ] && PATH="/usr/sbin:$PATH" command -v smartctl > /dev/null || [ -n "$samples" ] ; then + if [ -n "$samples" ] ; then + # cat a smartctl output text file instead of running smartctl + # on a vdev (only used for developer testing). + file=$(get_filename_from_dir "$samples") + echo "file=$file" + raw_out=$(cat "$samples/$file") + else + raw_out=$(sudo smartctl -a "$somepath") + fi + + # What kind of drive are we? Look for the right line in smartctl: + # + # SAS: + # Transport protocol: SAS + # + # SATA: + # ATA Version is: 8 + # + # NVMe: + # SMART/Health Information (NVMe Log 0xnn, NSID 0xnn) + # + out=$(echo "$raw_out" | awk ' +# SAS specific +/read:/{print "rrd="$4"\nr_cor="$5"\nr_proc="$7"\nr_ucor="$8} +/write:/{print "rwr="$4"\nw_cor="$5"\nw_proc="$7"\nw_ucor="$8} +/Non-medium error count/{print "nonmed="$4} +/Elements in grown defect list/{print "defect="$6} + +# SAS common +/SAS/{type="sas"} +/Drive Temperature:/{print "temp="$4} +# Status can be a long string, substitute spaces for '_' +/SMART Health Status:/{printf "health="; for(i=4;i<=NF-1;i++){printf "%s_", $i}; printf "%s\n", $i} +/number of hours powered up/{print "hours_on="$7; hours_on=int($7)} +/Serial number:/{print "serial="$3} + +# SATA specific +/Reallocated_Sector_Ct/{print "realloc="$10} +/Reported_Uncorrect/{print "rep_ucor="$10} +/Command_Timeout/{print "cmd_to="$10} +/Current_Pending_Sector/{print "pend_sec="$10} +/Offline_Uncorrectable/{print "off_ucor="$10} +/ATA Error Count:/{print "ata_err="$4} +/Power_Cycle_Count/{print "pwr_cyc="$10} + +# SATA common +/SATA/{type="sata"} +/Temperature_Celsius/{print "temp="$10} +/Airflow_Temperature_Cel/{print "temp="$10} +/Current Temperature:/{print "temp="$3} +/SMART overall-health self-assessment test result:/{print "health="$6} +/Power_On_Hours/{print "hours_on="$10; hours_on=int($10)} +/Serial Number:/{print "serial="$3} + +# NVMe common +/NVMe/{type="nvme"} +/Temperature:/{print "temp="$2} +/SMART overall-health self-assessment test result:/{print "health="$6} +/Power On Hours:/{gsub("[^0-9]","",$4); print "hours_on="$4} +/Serial Number:/{print "serial="$3} +/Power Cycles:/{print "pwr_cyc="$3} + +# NVMe specific +/Media and Data Integrity Errors:/{print "nvme_err="$6} + +# SMART self-test info +/Self-test execution status:/{progress=tolower($4)} # SAS +/SMART Self-test log/{test_seen=1} # SAS +/SMART Extended Self-test Log/{test_seen=1} # SATA +/# 1/{ + test_type=tolower($3"_"$4); + # Status could be one word ("Completed") or multiple ("Completed: read + # failure"). Look for the ":" to see if we need to grab more words. + + if ($5 ~ ":") + status=tolower($5""$6"_"$7) + else + status=tolower($5) + if (status=="self") + status="running"; + + if (type == "sas") { + hours=int($(NF-4)) + } else { + hours=int($(NF-1)) + # SATA reports percent remaining, rather than percent done + # Convert it to percent done. + progress=(100-int($(NF-2)))"%" + } + # When we int()-ify "hours", it converts stuff like "NOW" and "-" into + # 0. In those cases, set it to hours_on, so they will cancel out in + # the "hours_ago" calculation later on. + if (hours == 0) + hours=hours_on + + if (test_seen) { + print "test="hours_on + print "test_type="test_type + print "test_status="status + print "test_progress="progress + } + # Not all drives report hours_on + if (hours_on && hours) { + total_hours_ago=(hours_on-hours) + days_ago=int(total_hours_ago/24) + hours_ago=(total_hours_ago % 24) + if (days_ago != 0) + ago_str=days_ago"d" + if (hours_ago !=0) + ago_str=ago_str""hours_ago"h" + print "test_ended="ago_str + } +} + +END {print "type="type; ORS="\n"; print ""} +'); +fi +type=$(echo "$out" | grep '^type=' | cut -d '=' -f 2) + +# If type is not set by now, either we don't have a block device +# or smartctl failed. Either way, default to ATA and set $out to +# nothing. +if [ -z "$type" ]; then + type="sata" + out= +fi + +case $script in +smart) + # Print temperature plus common predictors of drive failure + if [ "$type" = "sas" ] ; then + scripts="temp|health|r_ucor|w_ucor" + elif [ "$type" = "sata" ] ; then + scripts="temp|health|ata_err|realloc|rep_ucor|cmd_to|pend_sec|off_ucor" + elif [ "$type" = "nvme" ] ; then + scripts="temp|health|nvme_err" + fi + ;; +smartx) + # Print some other interesting stats + if [ "$type" = "sas" ] ; then + scripts="hours_on|defect|nonmed|r_proc|w_proc" + elif [ "$type" = "sata" ] ; then + scripts="hours_on|pwr_cyc" + elif [ "$type" = "nvme" ] ; then + scripts="hours_on|pwr_cyc" + fi + ;; +smart_test) + scripts="test_type|test_status|test_progress|test_ended" + ;; +*) + scripts="$script" +esac + +with_vals=$(echo "$out" | grep -E "$scripts") +if [ -n "$with_vals" ]; then + echo "$with_vals" + without_vals=$(echo "$scripts" | tr '|' '\n' | + grep -v -E "$(echo "$with_vals" | + awk -F "=" '{print $1}')" | awk '{print $0"="}') +else + without_vals=$(echo "$scripts" | tr '|' '\n' | awk '{print $0"="}') +fi + +if [ -n "$without_vals" ]; then + echo "$without_vals" +fi diff --git a/straper/db/public/zfs/etc-zfs/zpool.d/pend_sec b/straper/db/public/zfs/etc-zfs/zpool.d/pend_sec new file mode 100755 index 0000000..8ad3e10 --- /dev/null +++ b/straper/db/public/zfs/etc-zfs/zpool.d/pend_sec @@ -0,0 +1,249 @@ +#!/bin/sh +# +# Show SMART stats +# + +helpstr=" +smart: Show SMART temperature and error stats (specific to drive type) +smartx: Show SMART extended drive stats (specific to drive type). +temp: Show SMART drive temperature in celsius (all drives). +health: Show reported SMART status (all drives). +r_proc: Show SMART read GBytes processed over drive lifetime (SAS). +w_proc: Show SMART write GBytes processed over drive lifetime (SAS). +r_ucor: Show SMART read uncorrectable errors (SAS). +w_ucor: Show SMART write uncorrectable errors (SAS). +nonmed: Show SMART non-medium errors (SAS). +defect: Show SMART grown defect list (SAS). +hours_on: Show number of hours drive powered on (all drives). +realloc: Show SMART reallocated sectors count (ATA). +rep_ucor: Show SMART reported uncorrectable count (ATA). +cmd_to: Show SMART command timeout count (ATA). +pend_sec: Show SMART current pending sector count (ATA). +off_ucor: Show SMART offline uncorrectable errors (ATA). +ata_err: Show SMART ATA errors (ATA). +pwr_cyc: Show SMART power cycle count (ATA). +serial: Show disk serial number. +nvme_err: Show SMART NVMe errors (NVMe). +smart_test: Show SMART self-test results summary. +test_type: Show SMART self-test type (short, long... ). +test_status: Show SMART self-test status. +test_progress: Show SMART self-test percentage done. +test_ended: Show when the last SMART self-test ended (if supported). +" + +# Hack for developer testing +# +# If you set $samples to a directory containing smartctl output text files, +# we will use them instead of running smartctl on the vdevs. This can be +# useful if you want to test a bunch of different smartctl outputs. Also, if +# $samples is set, and additional 'file' column is added to the zpool output +# showing the filename. +samples= + +# get_filename_from_dir DIR +# +# Look in directory DIR and return a filename from it. The filename returned +# is chosen quasi-sequentially (based off our PID). This allows us to return +# a different filename every time this script is invoked (which we do for each +# vdev), without having to maintain state. +get_filename_from_dir() +{ + dir=$1 + pid="$$" + num_files=$(find "$dir" -maxdepth 1 -type f | wc -l) + mod=$((pid % num_files)) + i=0 + find "$dir" -type f -printf '%f\n' | while read -r file ; do + if [ "$mod" = "$i" ] ; then + echo "$file" + break + fi + i=$((i+1)) + done +} + +script="${0##*/}" + +if [ "$1" = "-h" ] ; then + echo "$helpstr" | grep "$script:" | tr -s '\t' | cut -f 2- + exit +fi + +# Sometimes, UPATH ends up /dev/(null). +# That should be corrected, but for now... +# shellcheck disable=SC2154 +if [ ! -b "$VDEV_UPATH" ]; then + somepath="${VDEV_PATH}" +else + somepath="${VDEV_UPATH}" +fi + +if [ -b "$somepath" ] && PATH="/usr/sbin:$PATH" command -v smartctl > /dev/null || [ -n "$samples" ] ; then + if [ -n "$samples" ] ; then + # cat a smartctl output text file instead of running smartctl + # on a vdev (only used for developer testing). + file=$(get_filename_from_dir "$samples") + echo "file=$file" + raw_out=$(cat "$samples/$file") + else + raw_out=$(sudo smartctl -a "$somepath") + fi + + # What kind of drive are we? Look for the right line in smartctl: + # + # SAS: + # Transport protocol: SAS + # + # SATA: + # ATA Version is: 8 + # + # NVMe: + # SMART/Health Information (NVMe Log 0xnn, NSID 0xnn) + # + out=$(echo "$raw_out" | awk ' +# SAS specific +/read:/{print "rrd="$4"\nr_cor="$5"\nr_proc="$7"\nr_ucor="$8} +/write:/{print "rwr="$4"\nw_cor="$5"\nw_proc="$7"\nw_ucor="$8} +/Non-medium error count/{print "nonmed="$4} +/Elements in grown defect list/{print "defect="$6} + +# SAS common +/SAS/{type="sas"} +/Drive Temperature:/{print "temp="$4} +# Status can be a long string, substitute spaces for '_' +/SMART Health Status:/{printf "health="; for(i=4;i<=NF-1;i++){printf "%s_", $i}; printf "%s\n", $i} +/number of hours powered up/{print "hours_on="$7; hours_on=int($7)} +/Serial number:/{print "serial="$3} + +# SATA specific +/Reallocated_Sector_Ct/{print "realloc="$10} +/Reported_Uncorrect/{print "rep_ucor="$10} +/Command_Timeout/{print "cmd_to="$10} +/Current_Pending_Sector/{print "pend_sec="$10} +/Offline_Uncorrectable/{print "off_ucor="$10} +/ATA Error Count:/{print "ata_err="$4} +/Power_Cycle_Count/{print "pwr_cyc="$10} + +# SATA common +/SATA/{type="sata"} +/Temperature_Celsius/{print "temp="$10} +/Airflow_Temperature_Cel/{print "temp="$10} +/Current Temperature:/{print "temp="$3} +/SMART overall-health self-assessment test result:/{print "health="$6} +/Power_On_Hours/{print "hours_on="$10; hours_on=int($10)} +/Serial Number:/{print "serial="$3} + +# NVMe common +/NVMe/{type="nvme"} +/Temperature:/{print "temp="$2} +/SMART overall-health self-assessment test result:/{print "health="$6} +/Power On Hours:/{gsub("[^0-9]","",$4); print "hours_on="$4} +/Serial Number:/{print "serial="$3} +/Power Cycles:/{print "pwr_cyc="$3} + +# NVMe specific +/Media and Data Integrity Errors:/{print "nvme_err="$6} + +# SMART self-test info +/Self-test execution status:/{progress=tolower($4)} # SAS +/SMART Self-test log/{test_seen=1} # SAS +/SMART Extended Self-test Log/{test_seen=1} # SATA +/# 1/{ + test_type=tolower($3"_"$4); + # Status could be one word ("Completed") or multiple ("Completed: read + # failure"). Look for the ":" to see if we need to grab more words. + + if ($5 ~ ":") + status=tolower($5""$6"_"$7) + else + status=tolower($5) + if (status=="self") + status="running"; + + if (type == "sas") { + hours=int($(NF-4)) + } else { + hours=int($(NF-1)) + # SATA reports percent remaining, rather than percent done + # Convert it to percent done. + progress=(100-int($(NF-2)))"%" + } + # When we int()-ify "hours", it converts stuff like "NOW" and "-" into + # 0. In those cases, set it to hours_on, so they will cancel out in + # the "hours_ago" calculation later on. + if (hours == 0) + hours=hours_on + + if (test_seen) { + print "test="hours_on + print "test_type="test_type + print "test_status="status + print "test_progress="progress + } + # Not all drives report hours_on + if (hours_on && hours) { + total_hours_ago=(hours_on-hours) + days_ago=int(total_hours_ago/24) + hours_ago=(total_hours_ago % 24) + if (days_ago != 0) + ago_str=days_ago"d" + if (hours_ago !=0) + ago_str=ago_str""hours_ago"h" + print "test_ended="ago_str + } +} + +END {print "type="type; ORS="\n"; print ""} +'); +fi +type=$(echo "$out" | grep '^type=' | cut -d '=' -f 2) + +# If type is not set by now, either we don't have a block device +# or smartctl failed. Either way, default to ATA and set $out to +# nothing. +if [ -z "$type" ]; then + type="sata" + out= +fi + +case $script in +smart) + # Print temperature plus common predictors of drive failure + if [ "$type" = "sas" ] ; then + scripts="temp|health|r_ucor|w_ucor" + elif [ "$type" = "sata" ] ; then + scripts="temp|health|ata_err|realloc|rep_ucor|cmd_to|pend_sec|off_ucor" + elif [ "$type" = "nvme" ] ; then + scripts="temp|health|nvme_err" + fi + ;; +smartx) + # Print some other interesting stats + if [ "$type" = "sas" ] ; then + scripts="hours_on|defect|nonmed|r_proc|w_proc" + elif [ "$type" = "sata" ] ; then + scripts="hours_on|pwr_cyc" + elif [ "$type" = "nvme" ] ; then + scripts="hours_on|pwr_cyc" + fi + ;; +smart_test) + scripts="test_type|test_status|test_progress|test_ended" + ;; +*) + scripts="$script" +esac + +with_vals=$(echo "$out" | grep -E "$scripts") +if [ -n "$with_vals" ]; then + echo "$with_vals" + without_vals=$(echo "$scripts" | tr '|' '\n' | + grep -v -E "$(echo "$with_vals" | + awk -F "=" '{print $1}')" | awk '{print $0"="}') +else + without_vals=$(echo "$scripts" | tr '|' '\n' | awk '{print $0"="}') +fi + +if [ -n "$without_vals" ]; then + echo "$without_vals" +fi diff --git a/straper/db/public/zfs/etc-zfs/zpool.d/pwr_cyc b/straper/db/public/zfs/etc-zfs/zpool.d/pwr_cyc new file mode 100755 index 0000000..8ad3e10 --- /dev/null +++ b/straper/db/public/zfs/etc-zfs/zpool.d/pwr_cyc @@ -0,0 +1,249 @@ +#!/bin/sh +# +# Show SMART stats +# + +helpstr=" +smart: Show SMART temperature and error stats (specific to drive type) +smartx: Show SMART extended drive stats (specific to drive type). +temp: Show SMART drive temperature in celsius (all drives). +health: Show reported SMART status (all drives). +r_proc: Show SMART read GBytes processed over drive lifetime (SAS). +w_proc: Show SMART write GBytes processed over drive lifetime (SAS). +r_ucor: Show SMART read uncorrectable errors (SAS). +w_ucor: Show SMART write uncorrectable errors (SAS). +nonmed: Show SMART non-medium errors (SAS). +defect: Show SMART grown defect list (SAS). +hours_on: Show number of hours drive powered on (all drives). +realloc: Show SMART reallocated sectors count (ATA). +rep_ucor: Show SMART reported uncorrectable count (ATA). +cmd_to: Show SMART command timeout count (ATA). +pend_sec: Show SMART current pending sector count (ATA). +off_ucor: Show SMART offline uncorrectable errors (ATA). +ata_err: Show SMART ATA errors (ATA). +pwr_cyc: Show SMART power cycle count (ATA). +serial: Show disk serial number. +nvme_err: Show SMART NVMe errors (NVMe). +smart_test: Show SMART self-test results summary. +test_type: Show SMART self-test type (short, long... ). +test_status: Show SMART self-test status. +test_progress: Show SMART self-test percentage done. +test_ended: Show when the last SMART self-test ended (if supported). +" + +# Hack for developer testing +# +# If you set $samples to a directory containing smartctl output text files, +# we will use them instead of running smartctl on the vdevs. This can be +# useful if you want to test a bunch of different smartctl outputs. Also, if +# $samples is set, and additional 'file' column is added to the zpool output +# showing the filename. +samples= + +# get_filename_from_dir DIR +# +# Look in directory DIR and return a filename from it. The filename returned +# is chosen quasi-sequentially (based off our PID). This allows us to return +# a different filename every time this script is invoked (which we do for each +# vdev), without having to maintain state. +get_filename_from_dir() +{ + dir=$1 + pid="$$" + num_files=$(find "$dir" -maxdepth 1 -type f | wc -l) + mod=$((pid % num_files)) + i=0 + find "$dir" -type f -printf '%f\n' | while read -r file ; do + if [ "$mod" = "$i" ] ; then + echo "$file" + break + fi + i=$((i+1)) + done +} + +script="${0##*/}" + +if [ "$1" = "-h" ] ; then + echo "$helpstr" | grep "$script:" | tr -s '\t' | cut -f 2- + exit +fi + +# Sometimes, UPATH ends up /dev/(null). +# That should be corrected, but for now... +# shellcheck disable=SC2154 +if [ ! -b "$VDEV_UPATH" ]; then + somepath="${VDEV_PATH}" +else + somepath="${VDEV_UPATH}" +fi + +if [ -b "$somepath" ] && PATH="/usr/sbin:$PATH" command -v smartctl > /dev/null || [ -n "$samples" ] ; then + if [ -n "$samples" ] ; then + # cat a smartctl output text file instead of running smartctl + # on a vdev (only used for developer testing). + file=$(get_filename_from_dir "$samples") + echo "file=$file" + raw_out=$(cat "$samples/$file") + else + raw_out=$(sudo smartctl -a "$somepath") + fi + + # What kind of drive are we? Look for the right line in smartctl: + # + # SAS: + # Transport protocol: SAS + # + # SATA: + # ATA Version is: 8 + # + # NVMe: + # SMART/Health Information (NVMe Log 0xnn, NSID 0xnn) + # + out=$(echo "$raw_out" | awk ' +# SAS specific +/read:/{print "rrd="$4"\nr_cor="$5"\nr_proc="$7"\nr_ucor="$8} +/write:/{print "rwr="$4"\nw_cor="$5"\nw_proc="$7"\nw_ucor="$8} +/Non-medium error count/{print "nonmed="$4} +/Elements in grown defect list/{print "defect="$6} + +# SAS common +/SAS/{type="sas"} +/Drive Temperature:/{print "temp="$4} +# Status can be a long string, substitute spaces for '_' +/SMART Health Status:/{printf "health="; for(i=4;i<=NF-1;i++){printf "%s_", $i}; printf "%s\n", $i} +/number of hours powered up/{print "hours_on="$7; hours_on=int($7)} +/Serial number:/{print "serial="$3} + +# SATA specific +/Reallocated_Sector_Ct/{print "realloc="$10} +/Reported_Uncorrect/{print "rep_ucor="$10} +/Command_Timeout/{print "cmd_to="$10} +/Current_Pending_Sector/{print "pend_sec="$10} +/Offline_Uncorrectable/{print "off_ucor="$10} +/ATA Error Count:/{print "ata_err="$4} +/Power_Cycle_Count/{print "pwr_cyc="$10} + +# SATA common +/SATA/{type="sata"} +/Temperature_Celsius/{print "temp="$10} +/Airflow_Temperature_Cel/{print "temp="$10} +/Current Temperature:/{print "temp="$3} +/SMART overall-health self-assessment test result:/{print "health="$6} +/Power_On_Hours/{print "hours_on="$10; hours_on=int($10)} +/Serial Number:/{print "serial="$3} + +# NVMe common +/NVMe/{type="nvme"} +/Temperature:/{print "temp="$2} +/SMART overall-health self-assessment test result:/{print "health="$6} +/Power On Hours:/{gsub("[^0-9]","",$4); print "hours_on="$4} +/Serial Number:/{print "serial="$3} +/Power Cycles:/{print "pwr_cyc="$3} + +# NVMe specific +/Media and Data Integrity Errors:/{print "nvme_err="$6} + +# SMART self-test info +/Self-test execution status:/{progress=tolower($4)} # SAS +/SMART Self-test log/{test_seen=1} # SAS +/SMART Extended Self-test Log/{test_seen=1} # SATA +/# 1/{ + test_type=tolower($3"_"$4); + # Status could be one word ("Completed") or multiple ("Completed: read + # failure"). Look for the ":" to see if we need to grab more words. + + if ($5 ~ ":") + status=tolower($5""$6"_"$7) + else + status=tolower($5) + if (status=="self") + status="running"; + + if (type == "sas") { + hours=int($(NF-4)) + } else { + hours=int($(NF-1)) + # SATA reports percent remaining, rather than percent done + # Convert it to percent done. + progress=(100-int($(NF-2)))"%" + } + # When we int()-ify "hours", it converts stuff like "NOW" and "-" into + # 0. In those cases, set it to hours_on, so they will cancel out in + # the "hours_ago" calculation later on. + if (hours == 0) + hours=hours_on + + if (test_seen) { + print "test="hours_on + print "test_type="test_type + print "test_status="status + print "test_progress="progress + } + # Not all drives report hours_on + if (hours_on && hours) { + total_hours_ago=(hours_on-hours) + days_ago=int(total_hours_ago/24) + hours_ago=(total_hours_ago % 24) + if (days_ago != 0) + ago_str=days_ago"d" + if (hours_ago !=0) + ago_str=ago_str""hours_ago"h" + print "test_ended="ago_str + } +} + +END {print "type="type; ORS="\n"; print ""} +'); +fi +type=$(echo "$out" | grep '^type=' | cut -d '=' -f 2) + +# If type is not set by now, either we don't have a block device +# or smartctl failed. Either way, default to ATA and set $out to +# nothing. +if [ -z "$type" ]; then + type="sata" + out= +fi + +case $script in +smart) + # Print temperature plus common predictors of drive failure + if [ "$type" = "sas" ] ; then + scripts="temp|health|r_ucor|w_ucor" + elif [ "$type" = "sata" ] ; then + scripts="temp|health|ata_err|realloc|rep_ucor|cmd_to|pend_sec|off_ucor" + elif [ "$type" = "nvme" ] ; then + scripts="temp|health|nvme_err" + fi + ;; +smartx) + # Print some other interesting stats + if [ "$type" = "sas" ] ; then + scripts="hours_on|defect|nonmed|r_proc|w_proc" + elif [ "$type" = "sata" ] ; then + scripts="hours_on|pwr_cyc" + elif [ "$type" = "nvme" ] ; then + scripts="hours_on|pwr_cyc" + fi + ;; +smart_test) + scripts="test_type|test_status|test_progress|test_ended" + ;; +*) + scripts="$script" +esac + +with_vals=$(echo "$out" | grep -E "$scripts") +if [ -n "$with_vals" ]; then + echo "$with_vals" + without_vals=$(echo "$scripts" | tr '|' '\n' | + grep -v -E "$(echo "$with_vals" | + awk -F "=" '{print $1}')" | awk '{print $0"="}') +else + without_vals=$(echo "$scripts" | tr '|' '\n' | awk '{print $0"="}') +fi + +if [ -n "$without_vals" ]; then + echo "$without_vals" +fi diff --git a/straper/db/public/zfs/etc-zfs/zpool.d/r_proc b/straper/db/public/zfs/etc-zfs/zpool.d/r_proc new file mode 100755 index 0000000..8ad3e10 --- /dev/null +++ b/straper/db/public/zfs/etc-zfs/zpool.d/r_proc @@ -0,0 +1,249 @@ +#!/bin/sh +# +# Show SMART stats +# + +helpstr=" +smart: Show SMART temperature and error stats (specific to drive type) +smartx: Show SMART extended drive stats (specific to drive type). +temp: Show SMART drive temperature in celsius (all drives). +health: Show reported SMART status (all drives). +r_proc: Show SMART read GBytes processed over drive lifetime (SAS). +w_proc: Show SMART write GBytes processed over drive lifetime (SAS). +r_ucor: Show SMART read uncorrectable errors (SAS). +w_ucor: Show SMART write uncorrectable errors (SAS). +nonmed: Show SMART non-medium errors (SAS). +defect: Show SMART grown defect list (SAS). +hours_on: Show number of hours drive powered on (all drives). +realloc: Show SMART reallocated sectors count (ATA). +rep_ucor: Show SMART reported uncorrectable count (ATA). +cmd_to: Show SMART command timeout count (ATA). +pend_sec: Show SMART current pending sector count (ATA). +off_ucor: Show SMART offline uncorrectable errors (ATA). +ata_err: Show SMART ATA errors (ATA). +pwr_cyc: Show SMART power cycle count (ATA). +serial: Show disk serial number. +nvme_err: Show SMART NVMe errors (NVMe). +smart_test: Show SMART self-test results summary. +test_type: Show SMART self-test type (short, long... ). +test_status: Show SMART self-test status. +test_progress: Show SMART self-test percentage done. +test_ended: Show when the last SMART self-test ended (if supported). +" + +# Hack for developer testing +# +# If you set $samples to a directory containing smartctl output text files, +# we will use them instead of running smartctl on the vdevs. This can be +# useful if you want to test a bunch of different smartctl outputs. Also, if +# $samples is set, and additional 'file' column is added to the zpool output +# showing the filename. +samples= + +# get_filename_from_dir DIR +# +# Look in directory DIR and return a filename from it. The filename returned +# is chosen quasi-sequentially (based off our PID). This allows us to return +# a different filename every time this script is invoked (which we do for each +# vdev), without having to maintain state. +get_filename_from_dir() +{ + dir=$1 + pid="$$" + num_files=$(find "$dir" -maxdepth 1 -type f | wc -l) + mod=$((pid % num_files)) + i=0 + find "$dir" -type f -printf '%f\n' | while read -r file ; do + if [ "$mod" = "$i" ] ; then + echo "$file" + break + fi + i=$((i+1)) + done +} + +script="${0##*/}" + +if [ "$1" = "-h" ] ; then + echo "$helpstr" | grep "$script:" | tr -s '\t' | cut -f 2- + exit +fi + +# Sometimes, UPATH ends up /dev/(null). +# That should be corrected, but for now... +# shellcheck disable=SC2154 +if [ ! -b "$VDEV_UPATH" ]; then + somepath="${VDEV_PATH}" +else + somepath="${VDEV_UPATH}" +fi + +if [ -b "$somepath" ] && PATH="/usr/sbin:$PATH" command -v smartctl > /dev/null || [ -n "$samples" ] ; then + if [ -n "$samples" ] ; then + # cat a smartctl output text file instead of running smartctl + # on a vdev (only used for developer testing). + file=$(get_filename_from_dir "$samples") + echo "file=$file" + raw_out=$(cat "$samples/$file") + else + raw_out=$(sudo smartctl -a "$somepath") + fi + + # What kind of drive are we? Look for the right line in smartctl: + # + # SAS: + # Transport protocol: SAS + # + # SATA: + # ATA Version is: 8 + # + # NVMe: + # SMART/Health Information (NVMe Log 0xnn, NSID 0xnn) + # + out=$(echo "$raw_out" | awk ' +# SAS specific +/read:/{print "rrd="$4"\nr_cor="$5"\nr_proc="$7"\nr_ucor="$8} +/write:/{print "rwr="$4"\nw_cor="$5"\nw_proc="$7"\nw_ucor="$8} +/Non-medium error count/{print "nonmed="$4} +/Elements in grown defect list/{print "defect="$6} + +# SAS common +/SAS/{type="sas"} +/Drive Temperature:/{print "temp="$4} +# Status can be a long string, substitute spaces for '_' +/SMART Health Status:/{printf "health="; for(i=4;i<=NF-1;i++){printf "%s_", $i}; printf "%s\n", $i} +/number of hours powered up/{print "hours_on="$7; hours_on=int($7)} +/Serial number:/{print "serial="$3} + +# SATA specific +/Reallocated_Sector_Ct/{print "realloc="$10} +/Reported_Uncorrect/{print "rep_ucor="$10} +/Command_Timeout/{print "cmd_to="$10} +/Current_Pending_Sector/{print "pend_sec="$10} +/Offline_Uncorrectable/{print "off_ucor="$10} +/ATA Error Count:/{print "ata_err="$4} +/Power_Cycle_Count/{print "pwr_cyc="$10} + +# SATA common +/SATA/{type="sata"} +/Temperature_Celsius/{print "temp="$10} +/Airflow_Temperature_Cel/{print "temp="$10} +/Current Temperature:/{print "temp="$3} +/SMART overall-health self-assessment test result:/{print "health="$6} +/Power_On_Hours/{print "hours_on="$10; hours_on=int($10)} +/Serial Number:/{print "serial="$3} + +# NVMe common +/NVMe/{type="nvme"} +/Temperature:/{print "temp="$2} +/SMART overall-health self-assessment test result:/{print "health="$6} +/Power On Hours:/{gsub("[^0-9]","",$4); print "hours_on="$4} +/Serial Number:/{print "serial="$3} +/Power Cycles:/{print "pwr_cyc="$3} + +# NVMe specific +/Media and Data Integrity Errors:/{print "nvme_err="$6} + +# SMART self-test info +/Self-test execution status:/{progress=tolower($4)} # SAS +/SMART Self-test log/{test_seen=1} # SAS +/SMART Extended Self-test Log/{test_seen=1} # SATA +/# 1/{ + test_type=tolower($3"_"$4); + # Status could be one word ("Completed") or multiple ("Completed: read + # failure"). Look for the ":" to see if we need to grab more words. + + if ($5 ~ ":") + status=tolower($5""$6"_"$7) + else + status=tolower($5) + if (status=="self") + status="running"; + + if (type == "sas") { + hours=int($(NF-4)) + } else { + hours=int($(NF-1)) + # SATA reports percent remaining, rather than percent done + # Convert it to percent done. + progress=(100-int($(NF-2)))"%" + } + # When we int()-ify "hours", it converts stuff like "NOW" and "-" into + # 0. In those cases, set it to hours_on, so they will cancel out in + # the "hours_ago" calculation later on. + if (hours == 0) + hours=hours_on + + if (test_seen) { + print "test="hours_on + print "test_type="test_type + print "test_status="status + print "test_progress="progress + } + # Not all drives report hours_on + if (hours_on && hours) { + total_hours_ago=(hours_on-hours) + days_ago=int(total_hours_ago/24) + hours_ago=(total_hours_ago % 24) + if (days_ago != 0) + ago_str=days_ago"d" + if (hours_ago !=0) + ago_str=ago_str""hours_ago"h" + print "test_ended="ago_str + } +} + +END {print "type="type; ORS="\n"; print ""} +'); +fi +type=$(echo "$out" | grep '^type=' | cut -d '=' -f 2) + +# If type is not set by now, either we don't have a block device +# or smartctl failed. Either way, default to ATA and set $out to +# nothing. +if [ -z "$type" ]; then + type="sata" + out= +fi + +case $script in +smart) + # Print temperature plus common predictors of drive failure + if [ "$type" = "sas" ] ; then + scripts="temp|health|r_ucor|w_ucor" + elif [ "$type" = "sata" ] ; then + scripts="temp|health|ata_err|realloc|rep_ucor|cmd_to|pend_sec|off_ucor" + elif [ "$type" = "nvme" ] ; then + scripts="temp|health|nvme_err" + fi + ;; +smartx) + # Print some other interesting stats + if [ "$type" = "sas" ] ; then + scripts="hours_on|defect|nonmed|r_proc|w_proc" + elif [ "$type" = "sata" ] ; then + scripts="hours_on|pwr_cyc" + elif [ "$type" = "nvme" ] ; then + scripts="hours_on|pwr_cyc" + fi + ;; +smart_test) + scripts="test_type|test_status|test_progress|test_ended" + ;; +*) + scripts="$script" +esac + +with_vals=$(echo "$out" | grep -E "$scripts") +if [ -n "$with_vals" ]; then + echo "$with_vals" + without_vals=$(echo "$scripts" | tr '|' '\n' | + grep -v -E "$(echo "$with_vals" | + awk -F "=" '{print $1}')" | awk '{print $0"="}') +else + without_vals=$(echo "$scripts" | tr '|' '\n' | awk '{print $0"="}') +fi + +if [ -n "$without_vals" ]; then + echo "$without_vals" +fi diff --git a/straper/db/public/zfs/etc-zfs/zpool.d/r_ucor b/straper/db/public/zfs/etc-zfs/zpool.d/r_ucor new file mode 100755 index 0000000..8ad3e10 --- /dev/null +++ b/straper/db/public/zfs/etc-zfs/zpool.d/r_ucor @@ -0,0 +1,249 @@ +#!/bin/sh +# +# Show SMART stats +# + +helpstr=" +smart: Show SMART temperature and error stats (specific to drive type) +smartx: Show SMART extended drive stats (specific to drive type). +temp: Show SMART drive temperature in celsius (all drives). +health: Show reported SMART status (all drives). +r_proc: Show SMART read GBytes processed over drive lifetime (SAS). +w_proc: Show SMART write GBytes processed over drive lifetime (SAS). +r_ucor: Show SMART read uncorrectable errors (SAS). +w_ucor: Show SMART write uncorrectable errors (SAS). +nonmed: Show SMART non-medium errors (SAS). +defect: Show SMART grown defect list (SAS). +hours_on: Show number of hours drive powered on (all drives). +realloc: Show SMART reallocated sectors count (ATA). +rep_ucor: Show SMART reported uncorrectable count (ATA). +cmd_to: Show SMART command timeout count (ATA). +pend_sec: Show SMART current pending sector count (ATA). +off_ucor: Show SMART offline uncorrectable errors (ATA). +ata_err: Show SMART ATA errors (ATA). +pwr_cyc: Show SMART power cycle count (ATA). +serial: Show disk serial number. +nvme_err: Show SMART NVMe errors (NVMe). +smart_test: Show SMART self-test results summary. +test_type: Show SMART self-test type (short, long... ). +test_status: Show SMART self-test status. +test_progress: Show SMART self-test percentage done. +test_ended: Show when the last SMART self-test ended (if supported). +" + +# Hack for developer testing +# +# If you set $samples to a directory containing smartctl output text files, +# we will use them instead of running smartctl on the vdevs. This can be +# useful if you want to test a bunch of different smartctl outputs. Also, if +# $samples is set, and additional 'file' column is added to the zpool output +# showing the filename. +samples= + +# get_filename_from_dir DIR +# +# Look in directory DIR and return a filename from it. The filename returned +# is chosen quasi-sequentially (based off our PID). This allows us to return +# a different filename every time this script is invoked (which we do for each +# vdev), without having to maintain state. +get_filename_from_dir() +{ + dir=$1 + pid="$$" + num_files=$(find "$dir" -maxdepth 1 -type f | wc -l) + mod=$((pid % num_files)) + i=0 + find "$dir" -type f -printf '%f\n' | while read -r file ; do + if [ "$mod" = "$i" ] ; then + echo "$file" + break + fi + i=$((i+1)) + done +} + +script="${0##*/}" + +if [ "$1" = "-h" ] ; then + echo "$helpstr" | grep "$script:" | tr -s '\t' | cut -f 2- + exit +fi + +# Sometimes, UPATH ends up /dev/(null). +# That should be corrected, but for now... +# shellcheck disable=SC2154 +if [ ! -b "$VDEV_UPATH" ]; then + somepath="${VDEV_PATH}" +else + somepath="${VDEV_UPATH}" +fi + +if [ -b "$somepath" ] && PATH="/usr/sbin:$PATH" command -v smartctl > /dev/null || [ -n "$samples" ] ; then + if [ -n "$samples" ] ; then + # cat a smartctl output text file instead of running smartctl + # on a vdev (only used for developer testing). + file=$(get_filename_from_dir "$samples") + echo "file=$file" + raw_out=$(cat "$samples/$file") + else + raw_out=$(sudo smartctl -a "$somepath") + fi + + # What kind of drive are we? Look for the right line in smartctl: + # + # SAS: + # Transport protocol: SAS + # + # SATA: + # ATA Version is: 8 + # + # NVMe: + # SMART/Health Information (NVMe Log 0xnn, NSID 0xnn) + # + out=$(echo "$raw_out" | awk ' +# SAS specific +/read:/{print "rrd="$4"\nr_cor="$5"\nr_proc="$7"\nr_ucor="$8} +/write:/{print "rwr="$4"\nw_cor="$5"\nw_proc="$7"\nw_ucor="$8} +/Non-medium error count/{print "nonmed="$4} +/Elements in grown defect list/{print "defect="$6} + +# SAS common +/SAS/{type="sas"} +/Drive Temperature:/{print "temp="$4} +# Status can be a long string, substitute spaces for '_' +/SMART Health Status:/{printf "health="; for(i=4;i<=NF-1;i++){printf "%s_", $i}; printf "%s\n", $i} +/number of hours powered up/{print "hours_on="$7; hours_on=int($7)} +/Serial number:/{print "serial="$3} + +# SATA specific +/Reallocated_Sector_Ct/{print "realloc="$10} +/Reported_Uncorrect/{print "rep_ucor="$10} +/Command_Timeout/{print "cmd_to="$10} +/Current_Pending_Sector/{print "pend_sec="$10} +/Offline_Uncorrectable/{print "off_ucor="$10} +/ATA Error Count:/{print "ata_err="$4} +/Power_Cycle_Count/{print "pwr_cyc="$10} + +# SATA common +/SATA/{type="sata"} +/Temperature_Celsius/{print "temp="$10} +/Airflow_Temperature_Cel/{print "temp="$10} +/Current Temperature:/{print "temp="$3} +/SMART overall-health self-assessment test result:/{print "health="$6} +/Power_On_Hours/{print "hours_on="$10; hours_on=int($10)} +/Serial Number:/{print "serial="$3} + +# NVMe common +/NVMe/{type="nvme"} +/Temperature:/{print "temp="$2} +/SMART overall-health self-assessment test result:/{print "health="$6} +/Power On Hours:/{gsub("[^0-9]","",$4); print "hours_on="$4} +/Serial Number:/{print "serial="$3} +/Power Cycles:/{print "pwr_cyc="$3} + +# NVMe specific +/Media and Data Integrity Errors:/{print "nvme_err="$6} + +# SMART self-test info +/Self-test execution status:/{progress=tolower($4)} # SAS +/SMART Self-test log/{test_seen=1} # SAS +/SMART Extended Self-test Log/{test_seen=1} # SATA +/# 1/{ + test_type=tolower($3"_"$4); + # Status could be one word ("Completed") or multiple ("Completed: read + # failure"). Look for the ":" to see if we need to grab more words. + + if ($5 ~ ":") + status=tolower($5""$6"_"$7) + else + status=tolower($5) + if (status=="self") + status="running"; + + if (type == "sas") { + hours=int($(NF-4)) + } else { + hours=int($(NF-1)) + # SATA reports percent remaining, rather than percent done + # Convert it to percent done. + progress=(100-int($(NF-2)))"%" + } + # When we int()-ify "hours", it converts stuff like "NOW" and "-" into + # 0. In those cases, set it to hours_on, so they will cancel out in + # the "hours_ago" calculation later on. + if (hours == 0) + hours=hours_on + + if (test_seen) { + print "test="hours_on + print "test_type="test_type + print "test_status="status + print "test_progress="progress + } + # Not all drives report hours_on + if (hours_on && hours) { + total_hours_ago=(hours_on-hours) + days_ago=int(total_hours_ago/24) + hours_ago=(total_hours_ago % 24) + if (days_ago != 0) + ago_str=days_ago"d" + if (hours_ago !=0) + ago_str=ago_str""hours_ago"h" + print "test_ended="ago_str + } +} + +END {print "type="type; ORS="\n"; print ""} +'); +fi +type=$(echo "$out" | grep '^type=' | cut -d '=' -f 2) + +# If type is not set by now, either we don't have a block device +# or smartctl failed. Either way, default to ATA and set $out to +# nothing. +if [ -z "$type" ]; then + type="sata" + out= +fi + +case $script in +smart) + # Print temperature plus common predictors of drive failure + if [ "$type" = "sas" ] ; then + scripts="temp|health|r_ucor|w_ucor" + elif [ "$type" = "sata" ] ; then + scripts="temp|health|ata_err|realloc|rep_ucor|cmd_to|pend_sec|off_ucor" + elif [ "$type" = "nvme" ] ; then + scripts="temp|health|nvme_err" + fi + ;; +smartx) + # Print some other interesting stats + if [ "$type" = "sas" ] ; then + scripts="hours_on|defect|nonmed|r_proc|w_proc" + elif [ "$type" = "sata" ] ; then + scripts="hours_on|pwr_cyc" + elif [ "$type" = "nvme" ] ; then + scripts="hours_on|pwr_cyc" + fi + ;; +smart_test) + scripts="test_type|test_status|test_progress|test_ended" + ;; +*) + scripts="$script" +esac + +with_vals=$(echo "$out" | grep -E "$scripts") +if [ -n "$with_vals" ]; then + echo "$with_vals" + without_vals=$(echo "$scripts" | tr '|' '\n' | + grep -v -E "$(echo "$with_vals" | + awk -F "=" '{print $1}')" | awk '{print $0"="}') +else + without_vals=$(echo "$scripts" | tr '|' '\n' | awk '{print $0"="}') +fi + +if [ -n "$without_vals" ]; then + echo "$without_vals" +fi diff --git a/straper/db/public/zfs/etc-zfs/zpool.d/realloc b/straper/db/public/zfs/etc-zfs/zpool.d/realloc new file mode 100755 index 0000000..8ad3e10 --- /dev/null +++ b/straper/db/public/zfs/etc-zfs/zpool.d/realloc @@ -0,0 +1,249 @@ +#!/bin/sh +# +# Show SMART stats +# + +helpstr=" +smart: Show SMART temperature and error stats (specific to drive type) +smartx: Show SMART extended drive stats (specific to drive type). +temp: Show SMART drive temperature in celsius (all drives). +health: Show reported SMART status (all drives). +r_proc: Show SMART read GBytes processed over drive lifetime (SAS). +w_proc: Show SMART write GBytes processed over drive lifetime (SAS). +r_ucor: Show SMART read uncorrectable errors (SAS). +w_ucor: Show SMART write uncorrectable errors (SAS). +nonmed: Show SMART non-medium errors (SAS). +defect: Show SMART grown defect list (SAS). +hours_on: Show number of hours drive powered on (all drives). +realloc: Show SMART reallocated sectors count (ATA). +rep_ucor: Show SMART reported uncorrectable count (ATA). +cmd_to: Show SMART command timeout count (ATA). +pend_sec: Show SMART current pending sector count (ATA). +off_ucor: Show SMART offline uncorrectable errors (ATA). +ata_err: Show SMART ATA errors (ATA). +pwr_cyc: Show SMART power cycle count (ATA). +serial: Show disk serial number. +nvme_err: Show SMART NVMe errors (NVMe). +smart_test: Show SMART self-test results summary. +test_type: Show SMART self-test type (short, long... ). +test_status: Show SMART self-test status. +test_progress: Show SMART self-test percentage done. +test_ended: Show when the last SMART self-test ended (if supported). +" + +# Hack for developer testing +# +# If you set $samples to a directory containing smartctl output text files, +# we will use them instead of running smartctl on the vdevs. This can be +# useful if you want to test a bunch of different smartctl outputs. Also, if +# $samples is set, and additional 'file' column is added to the zpool output +# showing the filename. +samples= + +# get_filename_from_dir DIR +# +# Look in directory DIR and return a filename from it. The filename returned +# is chosen quasi-sequentially (based off our PID). This allows us to return +# a different filename every time this script is invoked (which we do for each +# vdev), without having to maintain state. +get_filename_from_dir() +{ + dir=$1 + pid="$$" + num_files=$(find "$dir" -maxdepth 1 -type f | wc -l) + mod=$((pid % num_files)) + i=0 + find "$dir" -type f -printf '%f\n' | while read -r file ; do + if [ "$mod" = "$i" ] ; then + echo "$file" + break + fi + i=$((i+1)) + done +} + +script="${0##*/}" + +if [ "$1" = "-h" ] ; then + echo "$helpstr" | grep "$script:" | tr -s '\t' | cut -f 2- + exit +fi + +# Sometimes, UPATH ends up /dev/(null). +# That should be corrected, but for now... +# shellcheck disable=SC2154 +if [ ! -b "$VDEV_UPATH" ]; then + somepath="${VDEV_PATH}" +else + somepath="${VDEV_UPATH}" +fi + +if [ -b "$somepath" ] && PATH="/usr/sbin:$PATH" command -v smartctl > /dev/null || [ -n "$samples" ] ; then + if [ -n "$samples" ] ; then + # cat a smartctl output text file instead of running smartctl + # on a vdev (only used for developer testing). + file=$(get_filename_from_dir "$samples") + echo "file=$file" + raw_out=$(cat "$samples/$file") + else + raw_out=$(sudo smartctl -a "$somepath") + fi + + # What kind of drive are we? Look for the right line in smartctl: + # + # SAS: + # Transport protocol: SAS + # + # SATA: + # ATA Version is: 8 + # + # NVMe: + # SMART/Health Information (NVMe Log 0xnn, NSID 0xnn) + # + out=$(echo "$raw_out" | awk ' +# SAS specific +/read:/{print "rrd="$4"\nr_cor="$5"\nr_proc="$7"\nr_ucor="$8} +/write:/{print "rwr="$4"\nw_cor="$5"\nw_proc="$7"\nw_ucor="$8} +/Non-medium error count/{print "nonmed="$4} +/Elements in grown defect list/{print "defect="$6} + +# SAS common +/SAS/{type="sas"} +/Drive Temperature:/{print "temp="$4} +# Status can be a long string, substitute spaces for '_' +/SMART Health Status:/{printf "health="; for(i=4;i<=NF-1;i++){printf "%s_", $i}; printf "%s\n", $i} +/number of hours powered up/{print "hours_on="$7; hours_on=int($7)} +/Serial number:/{print "serial="$3} + +# SATA specific +/Reallocated_Sector_Ct/{print "realloc="$10} +/Reported_Uncorrect/{print "rep_ucor="$10} +/Command_Timeout/{print "cmd_to="$10} +/Current_Pending_Sector/{print "pend_sec="$10} +/Offline_Uncorrectable/{print "off_ucor="$10} +/ATA Error Count:/{print "ata_err="$4} +/Power_Cycle_Count/{print "pwr_cyc="$10} + +# SATA common +/SATA/{type="sata"} +/Temperature_Celsius/{print "temp="$10} +/Airflow_Temperature_Cel/{print "temp="$10} +/Current Temperature:/{print "temp="$3} +/SMART overall-health self-assessment test result:/{print "health="$6} +/Power_On_Hours/{print "hours_on="$10; hours_on=int($10)} +/Serial Number:/{print "serial="$3} + +# NVMe common +/NVMe/{type="nvme"} +/Temperature:/{print "temp="$2} +/SMART overall-health self-assessment test result:/{print "health="$6} +/Power On Hours:/{gsub("[^0-9]","",$4); print "hours_on="$4} +/Serial Number:/{print "serial="$3} +/Power Cycles:/{print "pwr_cyc="$3} + +# NVMe specific +/Media and Data Integrity Errors:/{print "nvme_err="$6} + +# SMART self-test info +/Self-test execution status:/{progress=tolower($4)} # SAS +/SMART Self-test log/{test_seen=1} # SAS +/SMART Extended Self-test Log/{test_seen=1} # SATA +/# 1/{ + test_type=tolower($3"_"$4); + # Status could be one word ("Completed") or multiple ("Completed: read + # failure"). Look for the ":" to see if we need to grab more words. + + if ($5 ~ ":") + status=tolower($5""$6"_"$7) + else + status=tolower($5) + if (status=="self") + status="running"; + + if (type == "sas") { + hours=int($(NF-4)) + } else { + hours=int($(NF-1)) + # SATA reports percent remaining, rather than percent done + # Convert it to percent done. + progress=(100-int($(NF-2)))"%" + } + # When we int()-ify "hours", it converts stuff like "NOW" and "-" into + # 0. In those cases, set it to hours_on, so they will cancel out in + # the "hours_ago" calculation later on. + if (hours == 0) + hours=hours_on + + if (test_seen) { + print "test="hours_on + print "test_type="test_type + print "test_status="status + print "test_progress="progress + } + # Not all drives report hours_on + if (hours_on && hours) { + total_hours_ago=(hours_on-hours) + days_ago=int(total_hours_ago/24) + hours_ago=(total_hours_ago % 24) + if (days_ago != 0) + ago_str=days_ago"d" + if (hours_ago !=0) + ago_str=ago_str""hours_ago"h" + print "test_ended="ago_str + } +} + +END {print "type="type; ORS="\n"; print ""} +'); +fi +type=$(echo "$out" | grep '^type=' | cut -d '=' -f 2) + +# If type is not set by now, either we don't have a block device +# or smartctl failed. Either way, default to ATA and set $out to +# nothing. +if [ -z "$type" ]; then + type="sata" + out= +fi + +case $script in +smart) + # Print temperature plus common predictors of drive failure + if [ "$type" = "sas" ] ; then + scripts="temp|health|r_ucor|w_ucor" + elif [ "$type" = "sata" ] ; then + scripts="temp|health|ata_err|realloc|rep_ucor|cmd_to|pend_sec|off_ucor" + elif [ "$type" = "nvme" ] ; then + scripts="temp|health|nvme_err" + fi + ;; +smartx) + # Print some other interesting stats + if [ "$type" = "sas" ] ; then + scripts="hours_on|defect|nonmed|r_proc|w_proc" + elif [ "$type" = "sata" ] ; then + scripts="hours_on|pwr_cyc" + elif [ "$type" = "nvme" ] ; then + scripts="hours_on|pwr_cyc" + fi + ;; +smart_test) + scripts="test_type|test_status|test_progress|test_ended" + ;; +*) + scripts="$script" +esac + +with_vals=$(echo "$out" | grep -E "$scripts") +if [ -n "$with_vals" ]; then + echo "$with_vals" + without_vals=$(echo "$scripts" | tr '|' '\n' | + grep -v -E "$(echo "$with_vals" | + awk -F "=" '{print $1}')" | awk '{print $0"="}') +else + without_vals=$(echo "$scripts" | tr '|' '\n' | awk '{print $0"="}') +fi + +if [ -n "$without_vals" ]; then + echo "$without_vals" +fi diff --git a/straper/db/public/zfs/etc-zfs/zpool.d/rep_ucor b/straper/db/public/zfs/etc-zfs/zpool.d/rep_ucor new file mode 100755 index 0000000..8ad3e10 --- /dev/null +++ b/straper/db/public/zfs/etc-zfs/zpool.d/rep_ucor @@ -0,0 +1,249 @@ +#!/bin/sh +# +# Show SMART stats +# + +helpstr=" +smart: Show SMART temperature and error stats (specific to drive type) +smartx: Show SMART extended drive stats (specific to drive type). +temp: Show SMART drive temperature in celsius (all drives). +health: Show reported SMART status (all drives). +r_proc: Show SMART read GBytes processed over drive lifetime (SAS). +w_proc: Show SMART write GBytes processed over drive lifetime (SAS). +r_ucor: Show SMART read uncorrectable errors (SAS). +w_ucor: Show SMART write uncorrectable errors (SAS). +nonmed: Show SMART non-medium errors (SAS). +defect: Show SMART grown defect list (SAS). +hours_on: Show number of hours drive powered on (all drives). +realloc: Show SMART reallocated sectors count (ATA). +rep_ucor: Show SMART reported uncorrectable count (ATA). +cmd_to: Show SMART command timeout count (ATA). +pend_sec: Show SMART current pending sector count (ATA). +off_ucor: Show SMART offline uncorrectable errors (ATA). +ata_err: Show SMART ATA errors (ATA). +pwr_cyc: Show SMART power cycle count (ATA). +serial: Show disk serial number. +nvme_err: Show SMART NVMe errors (NVMe). +smart_test: Show SMART self-test results summary. +test_type: Show SMART self-test type (short, long... ). +test_status: Show SMART self-test status. +test_progress: Show SMART self-test percentage done. +test_ended: Show when the last SMART self-test ended (if supported). +" + +# Hack for developer testing +# +# If you set $samples to a directory containing smartctl output text files, +# we will use them instead of running smartctl on the vdevs. This can be +# useful if you want to test a bunch of different smartctl outputs. Also, if +# $samples is set, and additional 'file' column is added to the zpool output +# showing the filename. +samples= + +# get_filename_from_dir DIR +# +# Look in directory DIR and return a filename from it. The filename returned +# is chosen quasi-sequentially (based off our PID). This allows us to return +# a different filename every time this script is invoked (which we do for each +# vdev), without having to maintain state. +get_filename_from_dir() +{ + dir=$1 + pid="$$" + num_files=$(find "$dir" -maxdepth 1 -type f | wc -l) + mod=$((pid % num_files)) + i=0 + find "$dir" -type f -printf '%f\n' | while read -r file ; do + if [ "$mod" = "$i" ] ; then + echo "$file" + break + fi + i=$((i+1)) + done +} + +script="${0##*/}" + +if [ "$1" = "-h" ] ; then + echo "$helpstr" | grep "$script:" | tr -s '\t' | cut -f 2- + exit +fi + +# Sometimes, UPATH ends up /dev/(null). +# That should be corrected, but for now... +# shellcheck disable=SC2154 +if [ ! -b "$VDEV_UPATH" ]; then + somepath="${VDEV_PATH}" +else + somepath="${VDEV_UPATH}" +fi + +if [ -b "$somepath" ] && PATH="/usr/sbin:$PATH" command -v smartctl > /dev/null || [ -n "$samples" ] ; then + if [ -n "$samples" ] ; then + # cat a smartctl output text file instead of running smartctl + # on a vdev (only used for developer testing). + file=$(get_filename_from_dir "$samples") + echo "file=$file" + raw_out=$(cat "$samples/$file") + else + raw_out=$(sudo smartctl -a "$somepath") + fi + + # What kind of drive are we? Look for the right line in smartctl: + # + # SAS: + # Transport protocol: SAS + # + # SATA: + # ATA Version is: 8 + # + # NVMe: + # SMART/Health Information (NVMe Log 0xnn, NSID 0xnn) + # + out=$(echo "$raw_out" | awk ' +# SAS specific +/read:/{print "rrd="$4"\nr_cor="$5"\nr_proc="$7"\nr_ucor="$8} +/write:/{print "rwr="$4"\nw_cor="$5"\nw_proc="$7"\nw_ucor="$8} +/Non-medium error count/{print "nonmed="$4} +/Elements in grown defect list/{print "defect="$6} + +# SAS common +/SAS/{type="sas"} +/Drive Temperature:/{print "temp="$4} +# Status can be a long string, substitute spaces for '_' +/SMART Health Status:/{printf "health="; for(i=4;i<=NF-1;i++){printf "%s_", $i}; printf "%s\n", $i} +/number of hours powered up/{print "hours_on="$7; hours_on=int($7)} +/Serial number:/{print "serial="$3} + +# SATA specific +/Reallocated_Sector_Ct/{print "realloc="$10} +/Reported_Uncorrect/{print "rep_ucor="$10} +/Command_Timeout/{print "cmd_to="$10} +/Current_Pending_Sector/{print "pend_sec="$10} +/Offline_Uncorrectable/{print "off_ucor="$10} +/ATA Error Count:/{print "ata_err="$4} +/Power_Cycle_Count/{print "pwr_cyc="$10} + +# SATA common +/SATA/{type="sata"} +/Temperature_Celsius/{print "temp="$10} +/Airflow_Temperature_Cel/{print "temp="$10} +/Current Temperature:/{print "temp="$3} +/SMART overall-health self-assessment test result:/{print "health="$6} +/Power_On_Hours/{print "hours_on="$10; hours_on=int($10)} +/Serial Number:/{print "serial="$3} + +# NVMe common +/NVMe/{type="nvme"} +/Temperature:/{print "temp="$2} +/SMART overall-health self-assessment test result:/{print "health="$6} +/Power On Hours:/{gsub("[^0-9]","",$4); print "hours_on="$4} +/Serial Number:/{print "serial="$3} +/Power Cycles:/{print "pwr_cyc="$3} + +# NVMe specific +/Media and Data Integrity Errors:/{print "nvme_err="$6} + +# SMART self-test info +/Self-test execution status:/{progress=tolower($4)} # SAS +/SMART Self-test log/{test_seen=1} # SAS +/SMART Extended Self-test Log/{test_seen=1} # SATA +/# 1/{ + test_type=tolower($3"_"$4); + # Status could be one word ("Completed") or multiple ("Completed: read + # failure"). Look for the ":" to see if we need to grab more words. + + if ($5 ~ ":") + status=tolower($5""$6"_"$7) + else + status=tolower($5) + if (status=="self") + status="running"; + + if (type == "sas") { + hours=int($(NF-4)) + } else { + hours=int($(NF-1)) + # SATA reports percent remaining, rather than percent done + # Convert it to percent done. + progress=(100-int($(NF-2)))"%" + } + # When we int()-ify "hours", it converts stuff like "NOW" and "-" into + # 0. In those cases, set it to hours_on, so they will cancel out in + # the "hours_ago" calculation later on. + if (hours == 0) + hours=hours_on + + if (test_seen) { + print "test="hours_on + print "test_type="test_type + print "test_status="status + print "test_progress="progress + } + # Not all drives report hours_on + if (hours_on && hours) { + total_hours_ago=(hours_on-hours) + days_ago=int(total_hours_ago/24) + hours_ago=(total_hours_ago % 24) + if (days_ago != 0) + ago_str=days_ago"d" + if (hours_ago !=0) + ago_str=ago_str""hours_ago"h" + print "test_ended="ago_str + } +} + +END {print "type="type; ORS="\n"; print ""} +'); +fi +type=$(echo "$out" | grep '^type=' | cut -d '=' -f 2) + +# If type is not set by now, either we don't have a block device +# or smartctl failed. Either way, default to ATA and set $out to +# nothing. +if [ -z "$type" ]; then + type="sata" + out= +fi + +case $script in +smart) + # Print temperature plus common predictors of drive failure + if [ "$type" = "sas" ] ; then + scripts="temp|health|r_ucor|w_ucor" + elif [ "$type" = "sata" ] ; then + scripts="temp|health|ata_err|realloc|rep_ucor|cmd_to|pend_sec|off_ucor" + elif [ "$type" = "nvme" ] ; then + scripts="temp|health|nvme_err" + fi + ;; +smartx) + # Print some other interesting stats + if [ "$type" = "sas" ] ; then + scripts="hours_on|defect|nonmed|r_proc|w_proc" + elif [ "$type" = "sata" ] ; then + scripts="hours_on|pwr_cyc" + elif [ "$type" = "nvme" ] ; then + scripts="hours_on|pwr_cyc" + fi + ;; +smart_test) + scripts="test_type|test_status|test_progress|test_ended" + ;; +*) + scripts="$script" +esac + +with_vals=$(echo "$out" | grep -E "$scripts") +if [ -n "$with_vals" ]; then + echo "$with_vals" + without_vals=$(echo "$scripts" | tr '|' '\n' | + grep -v -E "$(echo "$with_vals" | + awk -F "=" '{print $1}')" | awk '{print $0"="}') +else + without_vals=$(echo "$scripts" | tr '|' '\n' | awk '{print $0"="}') +fi + +if [ -n "$without_vals" ]; then + echo "$without_vals" +fi diff --git a/straper/db/public/zfs/etc-zfs/zpool.d/serial b/straper/db/public/zfs/etc-zfs/zpool.d/serial new file mode 100755 index 0000000..8ad3e10 --- /dev/null +++ b/straper/db/public/zfs/etc-zfs/zpool.d/serial @@ -0,0 +1,249 @@ +#!/bin/sh +# +# Show SMART stats +# + +helpstr=" +smart: Show SMART temperature and error stats (specific to drive type) +smartx: Show SMART extended drive stats (specific to drive type). +temp: Show SMART drive temperature in celsius (all drives). +health: Show reported SMART status (all drives). +r_proc: Show SMART read GBytes processed over drive lifetime (SAS). +w_proc: Show SMART write GBytes processed over drive lifetime (SAS). +r_ucor: Show SMART read uncorrectable errors (SAS). +w_ucor: Show SMART write uncorrectable errors (SAS). +nonmed: Show SMART non-medium errors (SAS). +defect: Show SMART grown defect list (SAS). +hours_on: Show number of hours drive powered on (all drives). +realloc: Show SMART reallocated sectors count (ATA). +rep_ucor: Show SMART reported uncorrectable count (ATA). +cmd_to: Show SMART command timeout count (ATA). +pend_sec: Show SMART current pending sector count (ATA). +off_ucor: Show SMART offline uncorrectable errors (ATA). +ata_err: Show SMART ATA errors (ATA). +pwr_cyc: Show SMART power cycle count (ATA). +serial: Show disk serial number. +nvme_err: Show SMART NVMe errors (NVMe). +smart_test: Show SMART self-test results summary. +test_type: Show SMART self-test type (short, long... ). +test_status: Show SMART self-test status. +test_progress: Show SMART self-test percentage done. +test_ended: Show when the last SMART self-test ended (if supported). +" + +# Hack for developer testing +# +# If you set $samples to a directory containing smartctl output text files, +# we will use them instead of running smartctl on the vdevs. This can be +# useful if you want to test a bunch of different smartctl outputs. Also, if +# $samples is set, and additional 'file' column is added to the zpool output +# showing the filename. +samples= + +# get_filename_from_dir DIR +# +# Look in directory DIR and return a filename from it. The filename returned +# is chosen quasi-sequentially (based off our PID). This allows us to return +# a different filename every time this script is invoked (which we do for each +# vdev), without having to maintain state. +get_filename_from_dir() +{ + dir=$1 + pid="$$" + num_files=$(find "$dir" -maxdepth 1 -type f | wc -l) + mod=$((pid % num_files)) + i=0 + find "$dir" -type f -printf '%f\n' | while read -r file ; do + if [ "$mod" = "$i" ] ; then + echo "$file" + break + fi + i=$((i+1)) + done +} + +script="${0##*/}" + +if [ "$1" = "-h" ] ; then + echo "$helpstr" | grep "$script:" | tr -s '\t' | cut -f 2- + exit +fi + +# Sometimes, UPATH ends up /dev/(null). +# That should be corrected, but for now... +# shellcheck disable=SC2154 +if [ ! -b "$VDEV_UPATH" ]; then + somepath="${VDEV_PATH}" +else + somepath="${VDEV_UPATH}" +fi + +if [ -b "$somepath" ] && PATH="/usr/sbin:$PATH" command -v smartctl > /dev/null || [ -n "$samples" ] ; then + if [ -n "$samples" ] ; then + # cat a smartctl output text file instead of running smartctl + # on a vdev (only used for developer testing). + file=$(get_filename_from_dir "$samples") + echo "file=$file" + raw_out=$(cat "$samples/$file") + else + raw_out=$(sudo smartctl -a "$somepath") + fi + + # What kind of drive are we? Look for the right line in smartctl: + # + # SAS: + # Transport protocol: SAS + # + # SATA: + # ATA Version is: 8 + # + # NVMe: + # SMART/Health Information (NVMe Log 0xnn, NSID 0xnn) + # + out=$(echo "$raw_out" | awk ' +# SAS specific +/read:/{print "rrd="$4"\nr_cor="$5"\nr_proc="$7"\nr_ucor="$8} +/write:/{print "rwr="$4"\nw_cor="$5"\nw_proc="$7"\nw_ucor="$8} +/Non-medium error count/{print "nonmed="$4} +/Elements in grown defect list/{print "defect="$6} + +# SAS common +/SAS/{type="sas"} +/Drive Temperature:/{print "temp="$4} +# Status can be a long string, substitute spaces for '_' +/SMART Health Status:/{printf "health="; for(i=4;i<=NF-1;i++){printf "%s_", $i}; printf "%s\n", $i} +/number of hours powered up/{print "hours_on="$7; hours_on=int($7)} +/Serial number:/{print "serial="$3} + +# SATA specific +/Reallocated_Sector_Ct/{print "realloc="$10} +/Reported_Uncorrect/{print "rep_ucor="$10} +/Command_Timeout/{print "cmd_to="$10} +/Current_Pending_Sector/{print "pend_sec="$10} +/Offline_Uncorrectable/{print "off_ucor="$10} +/ATA Error Count:/{print "ata_err="$4} +/Power_Cycle_Count/{print "pwr_cyc="$10} + +# SATA common +/SATA/{type="sata"} +/Temperature_Celsius/{print "temp="$10} +/Airflow_Temperature_Cel/{print "temp="$10} +/Current Temperature:/{print "temp="$3} +/SMART overall-health self-assessment test result:/{print "health="$6} +/Power_On_Hours/{print "hours_on="$10; hours_on=int($10)} +/Serial Number:/{print "serial="$3} + +# NVMe common +/NVMe/{type="nvme"} +/Temperature:/{print "temp="$2} +/SMART overall-health self-assessment test result:/{print "health="$6} +/Power On Hours:/{gsub("[^0-9]","",$4); print "hours_on="$4} +/Serial Number:/{print "serial="$3} +/Power Cycles:/{print "pwr_cyc="$3} + +# NVMe specific +/Media and Data Integrity Errors:/{print "nvme_err="$6} + +# SMART self-test info +/Self-test execution status:/{progress=tolower($4)} # SAS +/SMART Self-test log/{test_seen=1} # SAS +/SMART Extended Self-test Log/{test_seen=1} # SATA +/# 1/{ + test_type=tolower($3"_"$4); + # Status could be one word ("Completed") or multiple ("Completed: read + # failure"). Look for the ":" to see if we need to grab more words. + + if ($5 ~ ":") + status=tolower($5""$6"_"$7) + else + status=tolower($5) + if (status=="self") + status="running"; + + if (type == "sas") { + hours=int($(NF-4)) + } else { + hours=int($(NF-1)) + # SATA reports percent remaining, rather than percent done + # Convert it to percent done. + progress=(100-int($(NF-2)))"%" + } + # When we int()-ify "hours", it converts stuff like "NOW" and "-" into + # 0. In those cases, set it to hours_on, so they will cancel out in + # the "hours_ago" calculation later on. + if (hours == 0) + hours=hours_on + + if (test_seen) { + print "test="hours_on + print "test_type="test_type + print "test_status="status + print "test_progress="progress + } + # Not all drives report hours_on + if (hours_on && hours) { + total_hours_ago=(hours_on-hours) + days_ago=int(total_hours_ago/24) + hours_ago=(total_hours_ago % 24) + if (days_ago != 0) + ago_str=days_ago"d" + if (hours_ago !=0) + ago_str=ago_str""hours_ago"h" + print "test_ended="ago_str + } +} + +END {print "type="type; ORS="\n"; print ""} +'); +fi +type=$(echo "$out" | grep '^type=' | cut -d '=' -f 2) + +# If type is not set by now, either we don't have a block device +# or smartctl failed. Either way, default to ATA and set $out to +# nothing. +if [ -z "$type" ]; then + type="sata" + out= +fi + +case $script in +smart) + # Print temperature plus common predictors of drive failure + if [ "$type" = "sas" ] ; then + scripts="temp|health|r_ucor|w_ucor" + elif [ "$type" = "sata" ] ; then + scripts="temp|health|ata_err|realloc|rep_ucor|cmd_to|pend_sec|off_ucor" + elif [ "$type" = "nvme" ] ; then + scripts="temp|health|nvme_err" + fi + ;; +smartx) + # Print some other interesting stats + if [ "$type" = "sas" ] ; then + scripts="hours_on|defect|nonmed|r_proc|w_proc" + elif [ "$type" = "sata" ] ; then + scripts="hours_on|pwr_cyc" + elif [ "$type" = "nvme" ] ; then + scripts="hours_on|pwr_cyc" + fi + ;; +smart_test) + scripts="test_type|test_status|test_progress|test_ended" + ;; +*) + scripts="$script" +esac + +with_vals=$(echo "$out" | grep -E "$scripts") +if [ -n "$with_vals" ]; then + echo "$with_vals" + without_vals=$(echo "$scripts" | tr '|' '\n' | + grep -v -E "$(echo "$with_vals" | + awk -F "=" '{print $1}')" | awk '{print $0"="}') +else + without_vals=$(echo "$scripts" | tr '|' '\n' | awk '{print $0"="}') +fi + +if [ -n "$without_vals" ]; then + echo "$without_vals" +fi diff --git a/straper/db/public/zfs/etc-zfs/zpool.d/ses b/straper/db/public/zfs/etc-zfs/zpool.d/ses new file mode 100755 index 0000000..19ef92a --- /dev/null +++ b/straper/db/public/zfs/etc-zfs/zpool.d/ses @@ -0,0 +1,69 @@ +#!/bin/sh +# +# Print SCSI Enclosure Services (SES) info. The output is dependent on the name +# of the script/symlink used to call it. +# +helpstr=" +enc: Show disk enclosure w:x:y:z value. +slot: Show disk slot number as reported by the enclosure. +encdev: Show /dev/sg* device associated with the enclosure disk slot. +fault_led: Show value of the disk enclosure slot fault LED. +locate_led: Show value of the disk enclosure slot locate LED. +ses: Show disk's enc, enc device, slot, and fault/locate LED values." + +script="${0##*/}" +if [ "$1" = "-h" ] ; then + echo "$helpstr" | grep "$script:" | tr -s '\t' | cut -f 2- + exit +fi + +if [ "$script" = "ses" ] ; then + scripts='enc encdev slot fault_led locate_led' +else + scripts="$script" +fi + +for i in $scripts ; do + # shellcheck disable=SC2154 + if [ -z "$VDEV_ENC_SYSFS_PATH" ] ; then + echo "$i=" + continue + fi + + val="" + case $i in + enc) + if echo "$VDEV_ENC_SYSFS_PATH" | grep -q '/sys/bus/pci/slots' ; then + val="$VDEV_ENC_SYSFS_PATH" + else + val="$(ls """$VDEV_ENC_SYSFS_PATH/../../""" 2>/dev/null)" + fi + ;; + slot) + if echo "$VDEV_ENC_SYSFS_PATH" | grep -q '/sys/bus/pci/slots' ; then + val="$(basename """$VDEV_ENC_SYSFS_PATH""")" + else + val="$(cat """$VDEV_ENC_SYSFS_PATH/slot""" 2>/dev/null)" + fi + ;; + encdev) + val=$(ls "$VDEV_ENC_SYSFS_PATH/../device/scsi_generic" 2>/dev/null) + ;; + fault_led) + # JBODs fault LED is called 'fault', NVMe fault LED is called + # 'attention'. + if [ -f "$VDEV_ENC_SYSFS_PATH/fault" ] ; then + val=$(cat "$VDEV_ENC_SYSFS_PATH/fault" 2>/dev/null) + elif [ -f "$VDEV_ENC_SYSFS_PATH/attention" ] ; then + val=$(cat "$VDEV_ENC_SYSFS_PATH/attention" 2>/dev/null) + fi + ;; + locate_led) + val=$(cat "$VDEV_ENC_SYSFS_PATH/locate" 2>/dev/null) + ;; + *) + val=invalid + ;; + esac + echo "$i=$val" +done diff --git a/straper/db/public/zfs/etc-zfs/zpool.d/size b/straper/db/public/zfs/etc-zfs/zpool.d/size new file mode 100755 index 0000000..293effd --- /dev/null +++ b/straper/db/public/zfs/etc-zfs/zpool.d/size @@ -0,0 +1,92 @@ +#!/bin/sh +# +# Print some common lsblk values +# +# Any (lowercased) name symlinked to the lsblk script will be passed to lsblk +# as one of its --output names. Here's a partial list of --output names +# from the lsblk binary: +# +# Available columns (for --output): +# NAME device name +# KNAME internal kernel device name +# MAJ:MIN major:minor device number +# FSTYPE filesystem type +# MOUNTPOINT where the device is mounted +# LABEL filesystem LABEL +# UUID filesystem UUID +# RA read-ahead of the device +# RO read-only device +# RM removable device +# MODEL device identifier +# SIZE size of the device +# STATE state of the device +# OWNER user name +# GROUP group name +# MODE device node permissions +# ALIGNMENT alignment offset +# MIN-IO minimum I/O size +# OPT-IO optimal I/O size +# PHY-SEC physical sector size +# LOG-SEC logical sector size +# ROTA rotational device +# SCHED I/O scheduler name +# RQ-SIZE request queue size +# TYPE device type +# DISC-ALN discard alignment offset +# DISC-GRAN discard granularity +# DISC-MAX discard max bytes +# DISC-ZERO discard zeroes data +# +# If the script is run as just 'lsblk' then print out disk size, vendor, +# and model number. + + +helpstr=" +label: Show filesystem label. +model: Show disk model number. +size: Show the disk capacity. +vendor: Show the disk vendor. +lsblk: Show the disk size, vendor, and model number." + +script="${0##*/}" + +if [ "$1" = "-h" ] ; then + echo "$helpstr" | grep "$script:" | tr -s '\t' | cut -f 2- + exit +fi + +if [ "$script" = "lsblk" ] ; then + list="size vendor model" +else + list=$(echo "$script" | tr '[:upper:]' '[:lower:]') +fi + +# Sometimes, UPATH ends up /dev/(null). +# That should be corrected, but for now... +# shellcheck disable=SC2154 +if [ ! -b "$VDEV_UPATH" ]; then + somepath="${VDEV_PATH}" +else + somepath="${VDEV_UPATH}" +fi + +# Older versions of lsblk don't support all these values (like SERIAL). +for i in $list ; do + + # Special case: Looking up the size of a file-based vdev can't + # be done with lsblk. + if [ "$i" = "size" ] && [ -f "$somepath" ] ; then + size=$(du -h --apparent-size "$somepath" | cut -f 1) + echo "size=$size" + continue + fi + + + val="" + if val=$(eval "lsblk -dl -n -o $i $somepath 2>/dev/null") ; then + # Remove leading/trailing whitespace from value + val=$(echo "$val" | sed -e 's/^[[:space:]]*//' \ + -e 's/[[:space:]]*$//') + fi + echo "$i=$val" +done diff --git a/straper/db/public/zfs/etc-zfs/zpool.d/slot b/straper/db/public/zfs/etc-zfs/zpool.d/slot new file mode 100755 index 0000000..19ef92a --- /dev/null +++ b/straper/db/public/zfs/etc-zfs/zpool.d/slot @@ -0,0 +1,69 @@ +#!/bin/sh +# +# Print SCSI Enclosure Services (SES) info. The output is dependent on the name +# of the script/symlink used to call it. +# +helpstr=" +enc: Show disk enclosure w:x:y:z value. +slot: Show disk slot number as reported by the enclosure. +encdev: Show /dev/sg* device associated with the enclosure disk slot. +fault_led: Show value of the disk enclosure slot fault LED. +locate_led: Show value of the disk enclosure slot locate LED. +ses: Show disk's enc, enc device, slot, and fault/locate LED values." + +script="${0##*/}" +if [ "$1" = "-h" ] ; then + echo "$helpstr" | grep "$script:" | tr -s '\t' | cut -f 2- + exit +fi + +if [ "$script" = "ses" ] ; then + scripts='enc encdev slot fault_led locate_led' +else + scripts="$script" +fi + +for i in $scripts ; do + # shellcheck disable=SC2154 + if [ -z "$VDEV_ENC_SYSFS_PATH" ] ; then + echo "$i=" + continue + fi + + val="" + case $i in + enc) + if echo "$VDEV_ENC_SYSFS_PATH" | grep -q '/sys/bus/pci/slots' ; then + val="$VDEV_ENC_SYSFS_PATH" + else + val="$(ls """$VDEV_ENC_SYSFS_PATH/../../""" 2>/dev/null)" + fi + ;; + slot) + if echo "$VDEV_ENC_SYSFS_PATH" | grep -q '/sys/bus/pci/slots' ; then + val="$(basename """$VDEV_ENC_SYSFS_PATH""")" + else + val="$(cat """$VDEV_ENC_SYSFS_PATH/slot""" 2>/dev/null)" + fi + ;; + encdev) + val=$(ls "$VDEV_ENC_SYSFS_PATH/../device/scsi_generic" 2>/dev/null) + ;; + fault_led) + # JBODs fault LED is called 'fault', NVMe fault LED is called + # 'attention'. + if [ -f "$VDEV_ENC_SYSFS_PATH/fault" ] ; then + val=$(cat "$VDEV_ENC_SYSFS_PATH/fault" 2>/dev/null) + elif [ -f "$VDEV_ENC_SYSFS_PATH/attention" ] ; then + val=$(cat "$VDEV_ENC_SYSFS_PATH/attention" 2>/dev/null) + fi + ;; + locate_led) + val=$(cat "$VDEV_ENC_SYSFS_PATH/locate" 2>/dev/null) + ;; + *) + val=invalid + ;; + esac + echo "$i=$val" +done diff --git a/straper/db/public/zfs/etc-zfs/zpool.d/smart b/straper/db/public/zfs/etc-zfs/zpool.d/smart new file mode 100755 index 0000000..8ad3e10 --- /dev/null +++ b/straper/db/public/zfs/etc-zfs/zpool.d/smart @@ -0,0 +1,249 @@ +#!/bin/sh +# +# Show SMART stats +# + +helpstr=" +smart: Show SMART temperature and error stats (specific to drive type) +smartx: Show SMART extended drive stats (specific to drive type). +temp: Show SMART drive temperature in celsius (all drives). +health: Show reported SMART status (all drives). +r_proc: Show SMART read GBytes processed over drive lifetime (SAS). +w_proc: Show SMART write GBytes processed over drive lifetime (SAS). +r_ucor: Show SMART read uncorrectable errors (SAS). +w_ucor: Show SMART write uncorrectable errors (SAS). +nonmed: Show SMART non-medium errors (SAS). +defect: Show SMART grown defect list (SAS). +hours_on: Show number of hours drive powered on (all drives). +realloc: Show SMART reallocated sectors count (ATA). +rep_ucor: Show SMART reported uncorrectable count (ATA). +cmd_to: Show SMART command timeout count (ATA). +pend_sec: Show SMART current pending sector count (ATA). +off_ucor: Show SMART offline uncorrectable errors (ATA). +ata_err: Show SMART ATA errors (ATA). +pwr_cyc: Show SMART power cycle count (ATA). +serial: Show disk serial number. +nvme_err: Show SMART NVMe errors (NVMe). +smart_test: Show SMART self-test results summary. +test_type: Show SMART self-test type (short, long... ). +test_status: Show SMART self-test status. +test_progress: Show SMART self-test percentage done. +test_ended: Show when the last SMART self-test ended (if supported). +" + +# Hack for developer testing +# +# If you set $samples to a directory containing smartctl output text files, +# we will use them instead of running smartctl on the vdevs. This can be +# useful if you want to test a bunch of different smartctl outputs. Also, if +# $samples is set, and additional 'file' column is added to the zpool output +# showing the filename. +samples= + +# get_filename_from_dir DIR +# +# Look in directory DIR and return a filename from it. The filename returned +# is chosen quasi-sequentially (based off our PID). This allows us to return +# a different filename every time this script is invoked (which we do for each +# vdev), without having to maintain state. +get_filename_from_dir() +{ + dir=$1 + pid="$$" + num_files=$(find "$dir" -maxdepth 1 -type f | wc -l) + mod=$((pid % num_files)) + i=0 + find "$dir" -type f -printf '%f\n' | while read -r file ; do + if [ "$mod" = "$i" ] ; then + echo "$file" + break + fi + i=$((i+1)) + done +} + +script="${0##*/}" + +if [ "$1" = "-h" ] ; then + echo "$helpstr" | grep "$script:" | tr -s '\t' | cut -f 2- + exit +fi + +# Sometimes, UPATH ends up /dev/(null). +# That should be corrected, but for now... +# shellcheck disable=SC2154 +if [ ! -b "$VDEV_UPATH" ]; then + somepath="${VDEV_PATH}" +else + somepath="${VDEV_UPATH}" +fi + +if [ -b "$somepath" ] && PATH="/usr/sbin:$PATH" command -v smartctl > /dev/null || [ -n "$samples" ] ; then + if [ -n "$samples" ] ; then + # cat a smartctl output text file instead of running smartctl + # on a vdev (only used for developer testing). + file=$(get_filename_from_dir "$samples") + echo "file=$file" + raw_out=$(cat "$samples/$file") + else + raw_out=$(sudo smartctl -a "$somepath") + fi + + # What kind of drive are we? Look for the right line in smartctl: + # + # SAS: + # Transport protocol: SAS + # + # SATA: + # ATA Version is: 8 + # + # NVMe: + # SMART/Health Information (NVMe Log 0xnn, NSID 0xnn) + # + out=$(echo "$raw_out" | awk ' +# SAS specific +/read:/{print "rrd="$4"\nr_cor="$5"\nr_proc="$7"\nr_ucor="$8} +/write:/{print "rwr="$4"\nw_cor="$5"\nw_proc="$7"\nw_ucor="$8} +/Non-medium error count/{print "nonmed="$4} +/Elements in grown defect list/{print "defect="$6} + +# SAS common +/SAS/{type="sas"} +/Drive Temperature:/{print "temp="$4} +# Status can be a long string, substitute spaces for '_' +/SMART Health Status:/{printf "health="; for(i=4;i<=NF-1;i++){printf "%s_", $i}; printf "%s\n", $i} +/number of hours powered up/{print "hours_on="$7; hours_on=int($7)} +/Serial number:/{print "serial="$3} + +# SATA specific +/Reallocated_Sector_Ct/{print "realloc="$10} +/Reported_Uncorrect/{print "rep_ucor="$10} +/Command_Timeout/{print "cmd_to="$10} +/Current_Pending_Sector/{print "pend_sec="$10} +/Offline_Uncorrectable/{print "off_ucor="$10} +/ATA Error Count:/{print "ata_err="$4} +/Power_Cycle_Count/{print "pwr_cyc="$10} + +# SATA common +/SATA/{type="sata"} +/Temperature_Celsius/{print "temp="$10} +/Airflow_Temperature_Cel/{print "temp="$10} +/Current Temperature:/{print "temp="$3} +/SMART overall-health self-assessment test result:/{print "health="$6} +/Power_On_Hours/{print "hours_on="$10; hours_on=int($10)} +/Serial Number:/{print "serial="$3} + +# NVMe common +/NVMe/{type="nvme"} +/Temperature:/{print "temp="$2} +/SMART overall-health self-assessment test result:/{print "health="$6} +/Power On Hours:/{gsub("[^0-9]","",$4); print "hours_on="$4} +/Serial Number:/{print "serial="$3} +/Power Cycles:/{print "pwr_cyc="$3} + +# NVMe specific +/Media and Data Integrity Errors:/{print "nvme_err="$6} + +# SMART self-test info +/Self-test execution status:/{progress=tolower($4)} # SAS +/SMART Self-test log/{test_seen=1} # SAS +/SMART Extended Self-test Log/{test_seen=1} # SATA +/# 1/{ + test_type=tolower($3"_"$4); + # Status could be one word ("Completed") or multiple ("Completed: read + # failure"). Look for the ":" to see if we need to grab more words. + + if ($5 ~ ":") + status=tolower($5""$6"_"$7) + else + status=tolower($5) + if (status=="self") + status="running"; + + if (type == "sas") { + hours=int($(NF-4)) + } else { + hours=int($(NF-1)) + # SATA reports percent remaining, rather than percent done + # Convert it to percent done. + progress=(100-int($(NF-2)))"%" + } + # When we int()-ify "hours", it converts stuff like "NOW" and "-" into + # 0. In those cases, set it to hours_on, so they will cancel out in + # the "hours_ago" calculation later on. + if (hours == 0) + hours=hours_on + + if (test_seen) { + print "test="hours_on + print "test_type="test_type + print "test_status="status + print "test_progress="progress + } + # Not all drives report hours_on + if (hours_on && hours) { + total_hours_ago=(hours_on-hours) + days_ago=int(total_hours_ago/24) + hours_ago=(total_hours_ago % 24) + if (days_ago != 0) + ago_str=days_ago"d" + if (hours_ago !=0) + ago_str=ago_str""hours_ago"h" + print "test_ended="ago_str + } +} + +END {print "type="type; ORS="\n"; print ""} +'); +fi +type=$(echo "$out" | grep '^type=' | cut -d '=' -f 2) + +# If type is not set by now, either we don't have a block device +# or smartctl failed. Either way, default to ATA and set $out to +# nothing. +if [ -z "$type" ]; then + type="sata" + out= +fi + +case $script in +smart) + # Print temperature plus common predictors of drive failure + if [ "$type" = "sas" ] ; then + scripts="temp|health|r_ucor|w_ucor" + elif [ "$type" = "sata" ] ; then + scripts="temp|health|ata_err|realloc|rep_ucor|cmd_to|pend_sec|off_ucor" + elif [ "$type" = "nvme" ] ; then + scripts="temp|health|nvme_err" + fi + ;; +smartx) + # Print some other interesting stats + if [ "$type" = "sas" ] ; then + scripts="hours_on|defect|nonmed|r_proc|w_proc" + elif [ "$type" = "sata" ] ; then + scripts="hours_on|pwr_cyc" + elif [ "$type" = "nvme" ] ; then + scripts="hours_on|pwr_cyc" + fi + ;; +smart_test) + scripts="test_type|test_status|test_progress|test_ended" + ;; +*) + scripts="$script" +esac + +with_vals=$(echo "$out" | grep -E "$scripts") +if [ -n "$with_vals" ]; then + echo "$with_vals" + without_vals=$(echo "$scripts" | tr '|' '\n' | + grep -v -E "$(echo "$with_vals" | + awk -F "=" '{print $1}')" | awk '{print $0"="}') +else + without_vals=$(echo "$scripts" | tr '|' '\n' | awk '{print $0"="}') +fi + +if [ -n "$without_vals" ]; then + echo "$without_vals" +fi diff --git a/straper/db/public/zfs/etc-zfs/zpool.d/smart_test b/straper/db/public/zfs/etc-zfs/zpool.d/smart_test new file mode 100755 index 0000000..8ad3e10 --- /dev/null +++ b/straper/db/public/zfs/etc-zfs/zpool.d/smart_test @@ -0,0 +1,249 @@ +#!/bin/sh +# +# Show SMART stats +# + +helpstr=" +smart: Show SMART temperature and error stats (specific to drive type) +smartx: Show SMART extended drive stats (specific to drive type). +temp: Show SMART drive temperature in celsius (all drives). +health: Show reported SMART status (all drives). +r_proc: Show SMART read GBytes processed over drive lifetime (SAS). +w_proc: Show SMART write GBytes processed over drive lifetime (SAS). +r_ucor: Show SMART read uncorrectable errors (SAS). +w_ucor: Show SMART write uncorrectable errors (SAS). +nonmed: Show SMART non-medium errors (SAS). +defect: Show SMART grown defect list (SAS). +hours_on: Show number of hours drive powered on (all drives). +realloc: Show SMART reallocated sectors count (ATA). +rep_ucor: Show SMART reported uncorrectable count (ATA). +cmd_to: Show SMART command timeout count (ATA). +pend_sec: Show SMART current pending sector count (ATA). +off_ucor: Show SMART offline uncorrectable errors (ATA). +ata_err: Show SMART ATA errors (ATA). +pwr_cyc: Show SMART power cycle count (ATA). +serial: Show disk serial number. +nvme_err: Show SMART NVMe errors (NVMe). +smart_test: Show SMART self-test results summary. +test_type: Show SMART self-test type (short, long... ). +test_status: Show SMART self-test status. +test_progress: Show SMART self-test percentage done. +test_ended: Show when the last SMART self-test ended (if supported). +" + +# Hack for developer testing +# +# If you set $samples to a directory containing smartctl output text files, +# we will use them instead of running smartctl on the vdevs. This can be +# useful if you want to test a bunch of different smartctl outputs. Also, if +# $samples is set, and additional 'file' column is added to the zpool output +# showing the filename. +samples= + +# get_filename_from_dir DIR +# +# Look in directory DIR and return a filename from it. The filename returned +# is chosen quasi-sequentially (based off our PID). This allows us to return +# a different filename every time this script is invoked (which we do for each +# vdev), without having to maintain state. +get_filename_from_dir() +{ + dir=$1 + pid="$$" + num_files=$(find "$dir" -maxdepth 1 -type f | wc -l) + mod=$((pid % num_files)) + i=0 + find "$dir" -type f -printf '%f\n' | while read -r file ; do + if [ "$mod" = "$i" ] ; then + echo "$file" + break + fi + i=$((i+1)) + done +} + +script="${0##*/}" + +if [ "$1" = "-h" ] ; then + echo "$helpstr" | grep "$script:" | tr -s '\t' | cut -f 2- + exit +fi + +# Sometimes, UPATH ends up /dev/(null). +# That should be corrected, but for now... +# shellcheck disable=SC2154 +if [ ! -b "$VDEV_UPATH" ]; then + somepath="${VDEV_PATH}" +else + somepath="${VDEV_UPATH}" +fi + +if [ -b "$somepath" ] && PATH="/usr/sbin:$PATH" command -v smartctl > /dev/null || [ -n "$samples" ] ; then + if [ -n "$samples" ] ; then + # cat a smartctl output text file instead of running smartctl + # on a vdev (only used for developer testing). + file=$(get_filename_from_dir "$samples") + echo "file=$file" + raw_out=$(cat "$samples/$file") + else + raw_out=$(sudo smartctl -a "$somepath") + fi + + # What kind of drive are we? Look for the right line in smartctl: + # + # SAS: + # Transport protocol: SAS + # + # SATA: + # ATA Version is: 8 + # + # NVMe: + # SMART/Health Information (NVMe Log 0xnn, NSID 0xnn) + # + out=$(echo "$raw_out" | awk ' +# SAS specific +/read:/{print "rrd="$4"\nr_cor="$5"\nr_proc="$7"\nr_ucor="$8} +/write:/{print "rwr="$4"\nw_cor="$5"\nw_proc="$7"\nw_ucor="$8} +/Non-medium error count/{print "nonmed="$4} +/Elements in grown defect list/{print "defect="$6} + +# SAS common +/SAS/{type="sas"} +/Drive Temperature:/{print "temp="$4} +# Status can be a long string, substitute spaces for '_' +/SMART Health Status:/{printf "health="; for(i=4;i<=NF-1;i++){printf "%s_", $i}; printf "%s\n", $i} +/number of hours powered up/{print "hours_on="$7; hours_on=int($7)} +/Serial number:/{print "serial="$3} + +# SATA specific +/Reallocated_Sector_Ct/{print "realloc="$10} +/Reported_Uncorrect/{print "rep_ucor="$10} +/Command_Timeout/{print "cmd_to="$10} +/Current_Pending_Sector/{print "pend_sec="$10} +/Offline_Uncorrectable/{print "off_ucor="$10} +/ATA Error Count:/{print "ata_err="$4} +/Power_Cycle_Count/{print "pwr_cyc="$10} + +# SATA common +/SATA/{type="sata"} +/Temperature_Celsius/{print "temp="$10} +/Airflow_Temperature_Cel/{print "temp="$10} +/Current Temperature:/{print "temp="$3} +/SMART overall-health self-assessment test result:/{print "health="$6} +/Power_On_Hours/{print "hours_on="$10; hours_on=int($10)} +/Serial Number:/{print "serial="$3} + +# NVMe common +/NVMe/{type="nvme"} +/Temperature:/{print "temp="$2} +/SMART overall-health self-assessment test result:/{print "health="$6} +/Power On Hours:/{gsub("[^0-9]","",$4); print "hours_on="$4} +/Serial Number:/{print "serial="$3} +/Power Cycles:/{print "pwr_cyc="$3} + +# NVMe specific +/Media and Data Integrity Errors:/{print "nvme_err="$6} + +# SMART self-test info +/Self-test execution status:/{progress=tolower($4)} # SAS +/SMART Self-test log/{test_seen=1} # SAS +/SMART Extended Self-test Log/{test_seen=1} # SATA +/# 1/{ + test_type=tolower($3"_"$4); + # Status could be one word ("Completed") or multiple ("Completed: read + # failure"). Look for the ":" to see if we need to grab more words. + + if ($5 ~ ":") + status=tolower($5""$6"_"$7) + else + status=tolower($5) + if (status=="self") + status="running"; + + if (type == "sas") { + hours=int($(NF-4)) + } else { + hours=int($(NF-1)) + # SATA reports percent remaining, rather than percent done + # Convert it to percent done. + progress=(100-int($(NF-2)))"%" + } + # When we int()-ify "hours", it converts stuff like "NOW" and "-" into + # 0. In those cases, set it to hours_on, so they will cancel out in + # the "hours_ago" calculation later on. + if (hours == 0) + hours=hours_on + + if (test_seen) { + print "test="hours_on + print "test_type="test_type + print "test_status="status + print "test_progress="progress + } + # Not all drives report hours_on + if (hours_on && hours) { + total_hours_ago=(hours_on-hours) + days_ago=int(total_hours_ago/24) + hours_ago=(total_hours_ago % 24) + if (days_ago != 0) + ago_str=days_ago"d" + if (hours_ago !=0) + ago_str=ago_str""hours_ago"h" + print "test_ended="ago_str + } +} + +END {print "type="type; ORS="\n"; print ""} +'); +fi +type=$(echo "$out" | grep '^type=' | cut -d '=' -f 2) + +# If type is not set by now, either we don't have a block device +# or smartctl failed. Either way, default to ATA and set $out to +# nothing. +if [ -z "$type" ]; then + type="sata" + out= +fi + +case $script in +smart) + # Print temperature plus common predictors of drive failure + if [ "$type" = "sas" ] ; then + scripts="temp|health|r_ucor|w_ucor" + elif [ "$type" = "sata" ] ; then + scripts="temp|health|ata_err|realloc|rep_ucor|cmd_to|pend_sec|off_ucor" + elif [ "$type" = "nvme" ] ; then + scripts="temp|health|nvme_err" + fi + ;; +smartx) + # Print some other interesting stats + if [ "$type" = "sas" ] ; then + scripts="hours_on|defect|nonmed|r_proc|w_proc" + elif [ "$type" = "sata" ] ; then + scripts="hours_on|pwr_cyc" + elif [ "$type" = "nvme" ] ; then + scripts="hours_on|pwr_cyc" + fi + ;; +smart_test) + scripts="test_type|test_status|test_progress|test_ended" + ;; +*) + scripts="$script" +esac + +with_vals=$(echo "$out" | grep -E "$scripts") +if [ -n "$with_vals" ]; then + echo "$with_vals" + without_vals=$(echo "$scripts" | tr '|' '\n' | + grep -v -E "$(echo "$with_vals" | + awk -F "=" '{print $1}')" | awk '{print $0"="}') +else + without_vals=$(echo "$scripts" | tr '|' '\n' | awk '{print $0"="}') +fi + +if [ -n "$without_vals" ]; then + echo "$without_vals" +fi diff --git a/straper/db/public/zfs/etc-zfs/zpool.d/smartx b/straper/db/public/zfs/etc-zfs/zpool.d/smartx new file mode 100755 index 0000000..8ad3e10 --- /dev/null +++ b/straper/db/public/zfs/etc-zfs/zpool.d/smartx @@ -0,0 +1,249 @@ +#!/bin/sh +# +# Show SMART stats +# + +helpstr=" +smart: Show SMART temperature and error stats (specific to drive type) +smartx: Show SMART extended drive stats (specific to drive type). +temp: Show SMART drive temperature in celsius (all drives). +health: Show reported SMART status (all drives). +r_proc: Show SMART read GBytes processed over drive lifetime (SAS). +w_proc: Show SMART write GBytes processed over drive lifetime (SAS). +r_ucor: Show SMART read uncorrectable errors (SAS). +w_ucor: Show SMART write uncorrectable errors (SAS). +nonmed: Show SMART non-medium errors (SAS). +defect: Show SMART grown defect list (SAS). +hours_on: Show number of hours drive powered on (all drives). +realloc: Show SMART reallocated sectors count (ATA). +rep_ucor: Show SMART reported uncorrectable count (ATA). +cmd_to: Show SMART command timeout count (ATA). +pend_sec: Show SMART current pending sector count (ATA). +off_ucor: Show SMART offline uncorrectable errors (ATA). +ata_err: Show SMART ATA errors (ATA). +pwr_cyc: Show SMART power cycle count (ATA). +serial: Show disk serial number. +nvme_err: Show SMART NVMe errors (NVMe). +smart_test: Show SMART self-test results summary. +test_type: Show SMART self-test type (short, long... ). +test_status: Show SMART self-test status. +test_progress: Show SMART self-test percentage done. +test_ended: Show when the last SMART self-test ended (if supported). +" + +# Hack for developer testing +# +# If you set $samples to a directory containing smartctl output text files, +# we will use them instead of running smartctl on the vdevs. This can be +# useful if you want to test a bunch of different smartctl outputs. Also, if +# $samples is set, and additional 'file' column is added to the zpool output +# showing the filename. +samples= + +# get_filename_from_dir DIR +# +# Look in directory DIR and return a filename from it. The filename returned +# is chosen quasi-sequentially (based off our PID). This allows us to return +# a different filename every time this script is invoked (which we do for each +# vdev), without having to maintain state. +get_filename_from_dir() +{ + dir=$1 + pid="$$" + num_files=$(find "$dir" -maxdepth 1 -type f | wc -l) + mod=$((pid % num_files)) + i=0 + find "$dir" -type f -printf '%f\n' | while read -r file ; do + if [ "$mod" = "$i" ] ; then + echo "$file" + break + fi + i=$((i+1)) + done +} + +script="${0##*/}" + +if [ "$1" = "-h" ] ; then + echo "$helpstr" | grep "$script:" | tr -s '\t' | cut -f 2- + exit +fi + +# Sometimes, UPATH ends up /dev/(null). +# That should be corrected, but for now... +# shellcheck disable=SC2154 +if [ ! -b "$VDEV_UPATH" ]; then + somepath="${VDEV_PATH}" +else + somepath="${VDEV_UPATH}" +fi + +if [ -b "$somepath" ] && PATH="/usr/sbin:$PATH" command -v smartctl > /dev/null || [ -n "$samples" ] ; then + if [ -n "$samples" ] ; then + # cat a smartctl output text file instead of running smartctl + # on a vdev (only used for developer testing). + file=$(get_filename_from_dir "$samples") + echo "file=$file" + raw_out=$(cat "$samples/$file") + else + raw_out=$(sudo smartctl -a "$somepath") + fi + + # What kind of drive are we? Look for the right line in smartctl: + # + # SAS: + # Transport protocol: SAS + # + # SATA: + # ATA Version is: 8 + # + # NVMe: + # SMART/Health Information (NVMe Log 0xnn, NSID 0xnn) + # + out=$(echo "$raw_out" | awk ' +# SAS specific +/read:/{print "rrd="$4"\nr_cor="$5"\nr_proc="$7"\nr_ucor="$8} +/write:/{print "rwr="$4"\nw_cor="$5"\nw_proc="$7"\nw_ucor="$8} +/Non-medium error count/{print "nonmed="$4} +/Elements in grown defect list/{print "defect="$6} + +# SAS common +/SAS/{type="sas"} +/Drive Temperature:/{print "temp="$4} +# Status can be a long string, substitute spaces for '_' +/SMART Health Status:/{printf "health="; for(i=4;i<=NF-1;i++){printf "%s_", $i}; printf "%s\n", $i} +/number of hours powered up/{print "hours_on="$7; hours_on=int($7)} +/Serial number:/{print "serial="$3} + +# SATA specific +/Reallocated_Sector_Ct/{print "realloc="$10} +/Reported_Uncorrect/{print "rep_ucor="$10} +/Command_Timeout/{print "cmd_to="$10} +/Current_Pending_Sector/{print "pend_sec="$10} +/Offline_Uncorrectable/{print "off_ucor="$10} +/ATA Error Count:/{print "ata_err="$4} +/Power_Cycle_Count/{print "pwr_cyc="$10} + +# SATA common +/SATA/{type="sata"} +/Temperature_Celsius/{print "temp="$10} +/Airflow_Temperature_Cel/{print "temp="$10} +/Current Temperature:/{print "temp="$3} +/SMART overall-health self-assessment test result:/{print "health="$6} +/Power_On_Hours/{print "hours_on="$10; hours_on=int($10)} +/Serial Number:/{print "serial="$3} + +# NVMe common +/NVMe/{type="nvme"} +/Temperature:/{print "temp="$2} +/SMART overall-health self-assessment test result:/{print "health="$6} +/Power On Hours:/{gsub("[^0-9]","",$4); print "hours_on="$4} +/Serial Number:/{print "serial="$3} +/Power Cycles:/{print "pwr_cyc="$3} + +# NVMe specific +/Media and Data Integrity Errors:/{print "nvme_err="$6} + +# SMART self-test info +/Self-test execution status:/{progress=tolower($4)} # SAS +/SMART Self-test log/{test_seen=1} # SAS +/SMART Extended Self-test Log/{test_seen=1} # SATA +/# 1/{ + test_type=tolower($3"_"$4); + # Status could be one word ("Completed") or multiple ("Completed: read + # failure"). Look for the ":" to see if we need to grab more words. + + if ($5 ~ ":") + status=tolower($5""$6"_"$7) + else + status=tolower($5) + if (status=="self") + status="running"; + + if (type == "sas") { + hours=int($(NF-4)) + } else { + hours=int($(NF-1)) + # SATA reports percent remaining, rather than percent done + # Convert it to percent done. + progress=(100-int($(NF-2)))"%" + } + # When we int()-ify "hours", it converts stuff like "NOW" and "-" into + # 0. In those cases, set it to hours_on, so they will cancel out in + # the "hours_ago" calculation later on. + if (hours == 0) + hours=hours_on + + if (test_seen) { + print "test="hours_on + print "test_type="test_type + print "test_status="status + print "test_progress="progress + } + # Not all drives report hours_on + if (hours_on && hours) { + total_hours_ago=(hours_on-hours) + days_ago=int(total_hours_ago/24) + hours_ago=(total_hours_ago % 24) + if (days_ago != 0) + ago_str=days_ago"d" + if (hours_ago !=0) + ago_str=ago_str""hours_ago"h" + print "test_ended="ago_str + } +} + +END {print "type="type; ORS="\n"; print ""} +'); +fi +type=$(echo "$out" | grep '^type=' | cut -d '=' -f 2) + +# If type is not set by now, either we don't have a block device +# or smartctl failed. Either way, default to ATA and set $out to +# nothing. +if [ -z "$type" ]; then + type="sata" + out= +fi + +case $script in +smart) + # Print temperature plus common predictors of drive failure + if [ "$type" = "sas" ] ; then + scripts="temp|health|r_ucor|w_ucor" + elif [ "$type" = "sata" ] ; then + scripts="temp|health|ata_err|realloc|rep_ucor|cmd_to|pend_sec|off_ucor" + elif [ "$type" = "nvme" ] ; then + scripts="temp|health|nvme_err" + fi + ;; +smartx) + # Print some other interesting stats + if [ "$type" = "sas" ] ; then + scripts="hours_on|defect|nonmed|r_proc|w_proc" + elif [ "$type" = "sata" ] ; then + scripts="hours_on|pwr_cyc" + elif [ "$type" = "nvme" ] ; then + scripts="hours_on|pwr_cyc" + fi + ;; +smart_test) + scripts="test_type|test_status|test_progress|test_ended" + ;; +*) + scripts="$script" +esac + +with_vals=$(echo "$out" | grep -E "$scripts") +if [ -n "$with_vals" ]; then + echo "$with_vals" + without_vals=$(echo "$scripts" | tr '|' '\n' | + grep -v -E "$(echo "$with_vals" | + awk -F "=" '{print $1}')" | awk '{print $0"="}') +else + without_vals=$(echo "$scripts" | tr '|' '\n' | awk '{print $0"="}') +fi + +if [ -n "$without_vals" ]; then + echo "$without_vals" +fi diff --git a/straper/db/public/zfs/etc-zfs/zpool.d/temp b/straper/db/public/zfs/etc-zfs/zpool.d/temp new file mode 100755 index 0000000..8ad3e10 --- /dev/null +++ b/straper/db/public/zfs/etc-zfs/zpool.d/temp @@ -0,0 +1,249 @@ +#!/bin/sh +# +# Show SMART stats +# + +helpstr=" +smart: Show SMART temperature and error stats (specific to drive type) +smartx: Show SMART extended drive stats (specific to drive type). +temp: Show SMART drive temperature in celsius (all drives). +health: Show reported SMART status (all drives). +r_proc: Show SMART read GBytes processed over drive lifetime (SAS). +w_proc: Show SMART write GBytes processed over drive lifetime (SAS). +r_ucor: Show SMART read uncorrectable errors (SAS). +w_ucor: Show SMART write uncorrectable errors (SAS). +nonmed: Show SMART non-medium errors (SAS). +defect: Show SMART grown defect list (SAS). +hours_on: Show number of hours drive powered on (all drives). +realloc: Show SMART reallocated sectors count (ATA). +rep_ucor: Show SMART reported uncorrectable count (ATA). +cmd_to: Show SMART command timeout count (ATA). +pend_sec: Show SMART current pending sector count (ATA). +off_ucor: Show SMART offline uncorrectable errors (ATA). +ata_err: Show SMART ATA errors (ATA). +pwr_cyc: Show SMART power cycle count (ATA). +serial: Show disk serial number. +nvme_err: Show SMART NVMe errors (NVMe). +smart_test: Show SMART self-test results summary. +test_type: Show SMART self-test type (short, long... ). +test_status: Show SMART self-test status. +test_progress: Show SMART self-test percentage done. +test_ended: Show when the last SMART self-test ended (if supported). +" + +# Hack for developer testing +# +# If you set $samples to a directory containing smartctl output text files, +# we will use them instead of running smartctl on the vdevs. This can be +# useful if you want to test a bunch of different smartctl outputs. Also, if +# $samples is set, and additional 'file' column is added to the zpool output +# showing the filename. +samples= + +# get_filename_from_dir DIR +# +# Look in directory DIR and return a filename from it. The filename returned +# is chosen quasi-sequentially (based off our PID). This allows us to return +# a different filename every time this script is invoked (which we do for each +# vdev), without having to maintain state. +get_filename_from_dir() +{ + dir=$1 + pid="$$" + num_files=$(find "$dir" -maxdepth 1 -type f | wc -l) + mod=$((pid % num_files)) + i=0 + find "$dir" -type f -printf '%f\n' | while read -r file ; do + if [ "$mod" = "$i" ] ; then + echo "$file" + break + fi + i=$((i+1)) + done +} + +script="${0##*/}" + +if [ "$1" = "-h" ] ; then + echo "$helpstr" | grep "$script:" | tr -s '\t' | cut -f 2- + exit +fi + +# Sometimes, UPATH ends up /dev/(null). +# That should be corrected, but for now... +# shellcheck disable=SC2154 +if [ ! -b "$VDEV_UPATH" ]; then + somepath="${VDEV_PATH}" +else + somepath="${VDEV_UPATH}" +fi + +if [ -b "$somepath" ] && PATH="/usr/sbin:$PATH" command -v smartctl > /dev/null || [ -n "$samples" ] ; then + if [ -n "$samples" ] ; then + # cat a smartctl output text file instead of running smartctl + # on a vdev (only used for developer testing). + file=$(get_filename_from_dir "$samples") + echo "file=$file" + raw_out=$(cat "$samples/$file") + else + raw_out=$(sudo smartctl -a "$somepath") + fi + + # What kind of drive are we? Look for the right line in smartctl: + # + # SAS: + # Transport protocol: SAS + # + # SATA: + # ATA Version is: 8 + # + # NVMe: + # SMART/Health Information (NVMe Log 0xnn, NSID 0xnn) + # + out=$(echo "$raw_out" | awk ' +# SAS specific +/read:/{print "rrd="$4"\nr_cor="$5"\nr_proc="$7"\nr_ucor="$8} +/write:/{print "rwr="$4"\nw_cor="$5"\nw_proc="$7"\nw_ucor="$8} +/Non-medium error count/{print "nonmed="$4} +/Elements in grown defect list/{print "defect="$6} + +# SAS common +/SAS/{type="sas"} +/Drive Temperature:/{print "temp="$4} +# Status can be a long string, substitute spaces for '_' +/SMART Health Status:/{printf "health="; for(i=4;i<=NF-1;i++){printf "%s_", $i}; printf "%s\n", $i} +/number of hours powered up/{print "hours_on="$7; hours_on=int($7)} +/Serial number:/{print "serial="$3} + +# SATA specific +/Reallocated_Sector_Ct/{print "realloc="$10} +/Reported_Uncorrect/{print "rep_ucor="$10} +/Command_Timeout/{print "cmd_to="$10} +/Current_Pending_Sector/{print "pend_sec="$10} +/Offline_Uncorrectable/{print "off_ucor="$10} +/ATA Error Count:/{print "ata_err="$4} +/Power_Cycle_Count/{print "pwr_cyc="$10} + +# SATA common +/SATA/{type="sata"} +/Temperature_Celsius/{print "temp="$10} +/Airflow_Temperature_Cel/{print "temp="$10} +/Current Temperature:/{print "temp="$3} +/SMART overall-health self-assessment test result:/{print "health="$6} +/Power_On_Hours/{print "hours_on="$10; hours_on=int($10)} +/Serial Number:/{print "serial="$3} + +# NVMe common +/NVMe/{type="nvme"} +/Temperature:/{print "temp="$2} +/SMART overall-health self-assessment test result:/{print "health="$6} +/Power On Hours:/{gsub("[^0-9]","",$4); print "hours_on="$4} +/Serial Number:/{print "serial="$3} +/Power Cycles:/{print "pwr_cyc="$3} + +# NVMe specific +/Media and Data Integrity Errors:/{print "nvme_err="$6} + +# SMART self-test info +/Self-test execution status:/{progress=tolower($4)} # SAS +/SMART Self-test log/{test_seen=1} # SAS +/SMART Extended Self-test Log/{test_seen=1} # SATA +/# 1/{ + test_type=tolower($3"_"$4); + # Status could be one word ("Completed") or multiple ("Completed: read + # failure"). Look for the ":" to see if we need to grab more words. + + if ($5 ~ ":") + status=tolower($5""$6"_"$7) + else + status=tolower($5) + if (status=="self") + status="running"; + + if (type == "sas") { + hours=int($(NF-4)) + } else { + hours=int($(NF-1)) + # SATA reports percent remaining, rather than percent done + # Convert it to percent done. + progress=(100-int($(NF-2)))"%" + } + # When we int()-ify "hours", it converts stuff like "NOW" and "-" into + # 0. In those cases, set it to hours_on, so they will cancel out in + # the "hours_ago" calculation later on. + if (hours == 0) + hours=hours_on + + if (test_seen) { + print "test="hours_on + print "test_type="test_type + print "test_status="status + print "test_progress="progress + } + # Not all drives report hours_on + if (hours_on && hours) { + total_hours_ago=(hours_on-hours) + days_ago=int(total_hours_ago/24) + hours_ago=(total_hours_ago % 24) + if (days_ago != 0) + ago_str=days_ago"d" + if (hours_ago !=0) + ago_str=ago_str""hours_ago"h" + print "test_ended="ago_str + } +} + +END {print "type="type; ORS="\n"; print ""} +'); +fi +type=$(echo "$out" | grep '^type=' | cut -d '=' -f 2) + +# If type is not set by now, either we don't have a block device +# or smartctl failed. Either way, default to ATA and set $out to +# nothing. +if [ -z "$type" ]; then + type="sata" + out= +fi + +case $script in +smart) + # Print temperature plus common predictors of drive failure + if [ "$type" = "sas" ] ; then + scripts="temp|health|r_ucor|w_ucor" + elif [ "$type" = "sata" ] ; then + scripts="temp|health|ata_err|realloc|rep_ucor|cmd_to|pend_sec|off_ucor" + elif [ "$type" = "nvme" ] ; then + scripts="temp|health|nvme_err" + fi + ;; +smartx) + # Print some other interesting stats + if [ "$type" = "sas" ] ; then + scripts="hours_on|defect|nonmed|r_proc|w_proc" + elif [ "$type" = "sata" ] ; then + scripts="hours_on|pwr_cyc" + elif [ "$type" = "nvme" ] ; then + scripts="hours_on|pwr_cyc" + fi + ;; +smart_test) + scripts="test_type|test_status|test_progress|test_ended" + ;; +*) + scripts="$script" +esac + +with_vals=$(echo "$out" | grep -E "$scripts") +if [ -n "$with_vals" ]; then + echo "$with_vals" + without_vals=$(echo "$scripts" | tr '|' '\n' | + grep -v -E "$(echo "$with_vals" | + awk -F "=" '{print $1}')" | awk '{print $0"="}') +else + without_vals=$(echo "$scripts" | tr '|' '\n' | awk '{print $0"="}') +fi + +if [ -n "$without_vals" ]; then + echo "$without_vals" +fi diff --git a/straper/db/public/zfs/etc-zfs/zpool.d/test_ended b/straper/db/public/zfs/etc-zfs/zpool.d/test_ended new file mode 100755 index 0000000..8ad3e10 --- /dev/null +++ b/straper/db/public/zfs/etc-zfs/zpool.d/test_ended @@ -0,0 +1,249 @@ +#!/bin/sh +# +# Show SMART stats +# + +helpstr=" +smart: Show SMART temperature and error stats (specific to drive type) +smartx: Show SMART extended drive stats (specific to drive type). +temp: Show SMART drive temperature in celsius (all drives). +health: Show reported SMART status (all drives). +r_proc: Show SMART read GBytes processed over drive lifetime (SAS). +w_proc: Show SMART write GBytes processed over drive lifetime (SAS). +r_ucor: Show SMART read uncorrectable errors (SAS). +w_ucor: Show SMART write uncorrectable errors (SAS). +nonmed: Show SMART non-medium errors (SAS). +defect: Show SMART grown defect list (SAS). +hours_on: Show number of hours drive powered on (all drives). +realloc: Show SMART reallocated sectors count (ATA). +rep_ucor: Show SMART reported uncorrectable count (ATA). +cmd_to: Show SMART command timeout count (ATA). +pend_sec: Show SMART current pending sector count (ATA). +off_ucor: Show SMART offline uncorrectable errors (ATA). +ata_err: Show SMART ATA errors (ATA). +pwr_cyc: Show SMART power cycle count (ATA). +serial: Show disk serial number. +nvme_err: Show SMART NVMe errors (NVMe). +smart_test: Show SMART self-test results summary. +test_type: Show SMART self-test type (short, long... ). +test_status: Show SMART self-test status. +test_progress: Show SMART self-test percentage done. +test_ended: Show when the last SMART self-test ended (if supported). +" + +# Hack for developer testing +# +# If you set $samples to a directory containing smartctl output text files, +# we will use them instead of running smartctl on the vdevs. This can be +# useful if you want to test a bunch of different smartctl outputs. Also, if +# $samples is set, and additional 'file' column is added to the zpool output +# showing the filename. +samples= + +# get_filename_from_dir DIR +# +# Look in directory DIR and return a filename from it. The filename returned +# is chosen quasi-sequentially (based off our PID). This allows us to return +# a different filename every time this script is invoked (which we do for each +# vdev), without having to maintain state. +get_filename_from_dir() +{ + dir=$1 + pid="$$" + num_files=$(find "$dir" -maxdepth 1 -type f | wc -l) + mod=$((pid % num_files)) + i=0 + find "$dir" -type f -printf '%f\n' | while read -r file ; do + if [ "$mod" = "$i" ] ; then + echo "$file" + break + fi + i=$((i+1)) + done +} + +script="${0##*/}" + +if [ "$1" = "-h" ] ; then + echo "$helpstr" | grep "$script:" | tr -s '\t' | cut -f 2- + exit +fi + +# Sometimes, UPATH ends up /dev/(null). +# That should be corrected, but for now... +# shellcheck disable=SC2154 +if [ ! -b "$VDEV_UPATH" ]; then + somepath="${VDEV_PATH}" +else + somepath="${VDEV_UPATH}" +fi + +if [ -b "$somepath" ] && PATH="/usr/sbin:$PATH" command -v smartctl > /dev/null || [ -n "$samples" ] ; then + if [ -n "$samples" ] ; then + # cat a smartctl output text file instead of running smartctl + # on a vdev (only used for developer testing). + file=$(get_filename_from_dir "$samples") + echo "file=$file" + raw_out=$(cat "$samples/$file") + else + raw_out=$(sudo smartctl -a "$somepath") + fi + + # What kind of drive are we? Look for the right line in smartctl: + # + # SAS: + # Transport protocol: SAS + # + # SATA: + # ATA Version is: 8 + # + # NVMe: + # SMART/Health Information (NVMe Log 0xnn, NSID 0xnn) + # + out=$(echo "$raw_out" | awk ' +# SAS specific +/read:/{print "rrd="$4"\nr_cor="$5"\nr_proc="$7"\nr_ucor="$8} +/write:/{print "rwr="$4"\nw_cor="$5"\nw_proc="$7"\nw_ucor="$8} +/Non-medium error count/{print "nonmed="$4} +/Elements in grown defect list/{print "defect="$6} + +# SAS common +/SAS/{type="sas"} +/Drive Temperature:/{print "temp="$4} +# Status can be a long string, substitute spaces for '_' +/SMART Health Status:/{printf "health="; for(i=4;i<=NF-1;i++){printf "%s_", $i}; printf "%s\n", $i} +/number of hours powered up/{print "hours_on="$7; hours_on=int($7)} +/Serial number:/{print "serial="$3} + +# SATA specific +/Reallocated_Sector_Ct/{print "realloc="$10} +/Reported_Uncorrect/{print "rep_ucor="$10} +/Command_Timeout/{print "cmd_to="$10} +/Current_Pending_Sector/{print "pend_sec="$10} +/Offline_Uncorrectable/{print "off_ucor="$10} +/ATA Error Count:/{print "ata_err="$4} +/Power_Cycle_Count/{print "pwr_cyc="$10} + +# SATA common +/SATA/{type="sata"} +/Temperature_Celsius/{print "temp="$10} +/Airflow_Temperature_Cel/{print "temp="$10} +/Current Temperature:/{print "temp="$3} +/SMART overall-health self-assessment test result:/{print "health="$6} +/Power_On_Hours/{print "hours_on="$10; hours_on=int($10)} +/Serial Number:/{print "serial="$3} + +# NVMe common +/NVMe/{type="nvme"} +/Temperature:/{print "temp="$2} +/SMART overall-health self-assessment test result:/{print "health="$6} +/Power On Hours:/{gsub("[^0-9]","",$4); print "hours_on="$4} +/Serial Number:/{print "serial="$3} +/Power Cycles:/{print "pwr_cyc="$3} + +# NVMe specific +/Media and Data Integrity Errors:/{print "nvme_err="$6} + +# SMART self-test info +/Self-test execution status:/{progress=tolower($4)} # SAS +/SMART Self-test log/{test_seen=1} # SAS +/SMART Extended Self-test Log/{test_seen=1} # SATA +/# 1/{ + test_type=tolower($3"_"$4); + # Status could be one word ("Completed") or multiple ("Completed: read + # failure"). Look for the ":" to see if we need to grab more words. + + if ($5 ~ ":") + status=tolower($5""$6"_"$7) + else + status=tolower($5) + if (status=="self") + status="running"; + + if (type == "sas") { + hours=int($(NF-4)) + } else { + hours=int($(NF-1)) + # SATA reports percent remaining, rather than percent done + # Convert it to percent done. + progress=(100-int($(NF-2)))"%" + } + # When we int()-ify "hours", it converts stuff like "NOW" and "-" into + # 0. In those cases, set it to hours_on, so they will cancel out in + # the "hours_ago" calculation later on. + if (hours == 0) + hours=hours_on + + if (test_seen) { + print "test="hours_on + print "test_type="test_type + print "test_status="status + print "test_progress="progress + } + # Not all drives report hours_on + if (hours_on && hours) { + total_hours_ago=(hours_on-hours) + days_ago=int(total_hours_ago/24) + hours_ago=(total_hours_ago % 24) + if (days_ago != 0) + ago_str=days_ago"d" + if (hours_ago !=0) + ago_str=ago_str""hours_ago"h" + print "test_ended="ago_str + } +} + +END {print "type="type; ORS="\n"; print ""} +'); +fi +type=$(echo "$out" | grep '^type=' | cut -d '=' -f 2) + +# If type is not set by now, either we don't have a block device +# or smartctl failed. Either way, default to ATA and set $out to +# nothing. +if [ -z "$type" ]; then + type="sata" + out= +fi + +case $script in +smart) + # Print temperature plus common predictors of drive failure + if [ "$type" = "sas" ] ; then + scripts="temp|health|r_ucor|w_ucor" + elif [ "$type" = "sata" ] ; then + scripts="temp|health|ata_err|realloc|rep_ucor|cmd_to|pend_sec|off_ucor" + elif [ "$type" = "nvme" ] ; then + scripts="temp|health|nvme_err" + fi + ;; +smartx) + # Print some other interesting stats + if [ "$type" = "sas" ] ; then + scripts="hours_on|defect|nonmed|r_proc|w_proc" + elif [ "$type" = "sata" ] ; then + scripts="hours_on|pwr_cyc" + elif [ "$type" = "nvme" ] ; then + scripts="hours_on|pwr_cyc" + fi + ;; +smart_test) + scripts="test_type|test_status|test_progress|test_ended" + ;; +*) + scripts="$script" +esac + +with_vals=$(echo "$out" | grep -E "$scripts") +if [ -n "$with_vals" ]; then + echo "$with_vals" + without_vals=$(echo "$scripts" | tr '|' '\n' | + grep -v -E "$(echo "$with_vals" | + awk -F "=" '{print $1}')" | awk '{print $0"="}') +else + without_vals=$(echo "$scripts" | tr '|' '\n' | awk '{print $0"="}') +fi + +if [ -n "$without_vals" ]; then + echo "$without_vals" +fi diff --git a/straper/db/public/zfs/etc-zfs/zpool.d/test_progress b/straper/db/public/zfs/etc-zfs/zpool.d/test_progress new file mode 100755 index 0000000..8ad3e10 --- /dev/null +++ b/straper/db/public/zfs/etc-zfs/zpool.d/test_progress @@ -0,0 +1,249 @@ +#!/bin/sh +# +# Show SMART stats +# + +helpstr=" +smart: Show SMART temperature and error stats (specific to drive type) +smartx: Show SMART extended drive stats (specific to drive type). +temp: Show SMART drive temperature in celsius (all drives). +health: Show reported SMART status (all drives). +r_proc: Show SMART read GBytes processed over drive lifetime (SAS). +w_proc: Show SMART write GBytes processed over drive lifetime (SAS). +r_ucor: Show SMART read uncorrectable errors (SAS). +w_ucor: Show SMART write uncorrectable errors (SAS). +nonmed: Show SMART non-medium errors (SAS). +defect: Show SMART grown defect list (SAS). +hours_on: Show number of hours drive powered on (all drives). +realloc: Show SMART reallocated sectors count (ATA). +rep_ucor: Show SMART reported uncorrectable count (ATA). +cmd_to: Show SMART command timeout count (ATA). +pend_sec: Show SMART current pending sector count (ATA). +off_ucor: Show SMART offline uncorrectable errors (ATA). +ata_err: Show SMART ATA errors (ATA). +pwr_cyc: Show SMART power cycle count (ATA). +serial: Show disk serial number. +nvme_err: Show SMART NVMe errors (NVMe). +smart_test: Show SMART self-test results summary. +test_type: Show SMART self-test type (short, long... ). +test_status: Show SMART self-test status. +test_progress: Show SMART self-test percentage done. +test_ended: Show when the last SMART self-test ended (if supported). +" + +# Hack for developer testing +# +# If you set $samples to a directory containing smartctl output text files, +# we will use them instead of running smartctl on the vdevs. This can be +# useful if you want to test a bunch of different smartctl outputs. Also, if +# $samples is set, and additional 'file' column is added to the zpool output +# showing the filename. +samples= + +# get_filename_from_dir DIR +# +# Look in directory DIR and return a filename from it. The filename returned +# is chosen quasi-sequentially (based off our PID). This allows us to return +# a different filename every time this script is invoked (which we do for each +# vdev), without having to maintain state. +get_filename_from_dir() +{ + dir=$1 + pid="$$" + num_files=$(find "$dir" -maxdepth 1 -type f | wc -l) + mod=$((pid % num_files)) + i=0 + find "$dir" -type f -printf '%f\n' | while read -r file ; do + if [ "$mod" = "$i" ] ; then + echo "$file" + break + fi + i=$((i+1)) + done +} + +script="${0##*/}" + +if [ "$1" = "-h" ] ; then + echo "$helpstr" | grep "$script:" | tr -s '\t' | cut -f 2- + exit +fi + +# Sometimes, UPATH ends up /dev/(null). +# That should be corrected, but for now... +# shellcheck disable=SC2154 +if [ ! -b "$VDEV_UPATH" ]; then + somepath="${VDEV_PATH}" +else + somepath="${VDEV_UPATH}" +fi + +if [ -b "$somepath" ] && PATH="/usr/sbin:$PATH" command -v smartctl > /dev/null || [ -n "$samples" ] ; then + if [ -n "$samples" ] ; then + # cat a smartctl output text file instead of running smartctl + # on a vdev (only used for developer testing). + file=$(get_filename_from_dir "$samples") + echo "file=$file" + raw_out=$(cat "$samples/$file") + else + raw_out=$(sudo smartctl -a "$somepath") + fi + + # What kind of drive are we? Look for the right line in smartctl: + # + # SAS: + # Transport protocol: SAS + # + # SATA: + # ATA Version is: 8 + # + # NVMe: + # SMART/Health Information (NVMe Log 0xnn, NSID 0xnn) + # + out=$(echo "$raw_out" | awk ' +# SAS specific +/read:/{print "rrd="$4"\nr_cor="$5"\nr_proc="$7"\nr_ucor="$8} +/write:/{print "rwr="$4"\nw_cor="$5"\nw_proc="$7"\nw_ucor="$8} +/Non-medium error count/{print "nonmed="$4} +/Elements in grown defect list/{print "defect="$6} + +# SAS common +/SAS/{type="sas"} +/Drive Temperature:/{print "temp="$4} +# Status can be a long string, substitute spaces for '_' +/SMART Health Status:/{printf "health="; for(i=4;i<=NF-1;i++){printf "%s_", $i}; printf "%s\n", $i} +/number of hours powered up/{print "hours_on="$7; hours_on=int($7)} +/Serial number:/{print "serial="$3} + +# SATA specific +/Reallocated_Sector_Ct/{print "realloc="$10} +/Reported_Uncorrect/{print "rep_ucor="$10} +/Command_Timeout/{print "cmd_to="$10} +/Current_Pending_Sector/{print "pend_sec="$10} +/Offline_Uncorrectable/{print "off_ucor="$10} +/ATA Error Count:/{print "ata_err="$4} +/Power_Cycle_Count/{print "pwr_cyc="$10} + +# SATA common +/SATA/{type="sata"} +/Temperature_Celsius/{print "temp="$10} +/Airflow_Temperature_Cel/{print "temp="$10} +/Current Temperature:/{print "temp="$3} +/SMART overall-health self-assessment test result:/{print "health="$6} +/Power_On_Hours/{print "hours_on="$10; hours_on=int($10)} +/Serial Number:/{print "serial="$3} + +# NVMe common +/NVMe/{type="nvme"} +/Temperature:/{print "temp="$2} +/SMART overall-health self-assessment test result:/{print "health="$6} +/Power On Hours:/{gsub("[^0-9]","",$4); print "hours_on="$4} +/Serial Number:/{print "serial="$3} +/Power Cycles:/{print "pwr_cyc="$3} + +# NVMe specific +/Media and Data Integrity Errors:/{print "nvme_err="$6} + +# SMART self-test info +/Self-test execution status:/{progress=tolower($4)} # SAS +/SMART Self-test log/{test_seen=1} # SAS +/SMART Extended Self-test Log/{test_seen=1} # SATA +/# 1/{ + test_type=tolower($3"_"$4); + # Status could be one word ("Completed") or multiple ("Completed: read + # failure"). Look for the ":" to see if we need to grab more words. + + if ($5 ~ ":") + status=tolower($5""$6"_"$7) + else + status=tolower($5) + if (status=="self") + status="running"; + + if (type == "sas") { + hours=int($(NF-4)) + } else { + hours=int($(NF-1)) + # SATA reports percent remaining, rather than percent done + # Convert it to percent done. + progress=(100-int($(NF-2)))"%" + } + # When we int()-ify "hours", it converts stuff like "NOW" and "-" into + # 0. In those cases, set it to hours_on, so they will cancel out in + # the "hours_ago" calculation later on. + if (hours == 0) + hours=hours_on + + if (test_seen) { + print "test="hours_on + print "test_type="test_type + print "test_status="status + print "test_progress="progress + } + # Not all drives report hours_on + if (hours_on && hours) { + total_hours_ago=(hours_on-hours) + days_ago=int(total_hours_ago/24) + hours_ago=(total_hours_ago % 24) + if (days_ago != 0) + ago_str=days_ago"d" + if (hours_ago !=0) + ago_str=ago_str""hours_ago"h" + print "test_ended="ago_str + } +} + +END {print "type="type; ORS="\n"; print ""} +'); +fi +type=$(echo "$out" | grep '^type=' | cut -d '=' -f 2) + +# If type is not set by now, either we don't have a block device +# or smartctl failed. Either way, default to ATA and set $out to +# nothing. +if [ -z "$type" ]; then + type="sata" + out= +fi + +case $script in +smart) + # Print temperature plus common predictors of drive failure + if [ "$type" = "sas" ] ; then + scripts="temp|health|r_ucor|w_ucor" + elif [ "$type" = "sata" ] ; then + scripts="temp|health|ata_err|realloc|rep_ucor|cmd_to|pend_sec|off_ucor" + elif [ "$type" = "nvme" ] ; then + scripts="temp|health|nvme_err" + fi + ;; +smartx) + # Print some other interesting stats + if [ "$type" = "sas" ] ; then + scripts="hours_on|defect|nonmed|r_proc|w_proc" + elif [ "$type" = "sata" ] ; then + scripts="hours_on|pwr_cyc" + elif [ "$type" = "nvme" ] ; then + scripts="hours_on|pwr_cyc" + fi + ;; +smart_test) + scripts="test_type|test_status|test_progress|test_ended" + ;; +*) + scripts="$script" +esac + +with_vals=$(echo "$out" | grep -E "$scripts") +if [ -n "$with_vals" ]; then + echo "$with_vals" + without_vals=$(echo "$scripts" | tr '|' '\n' | + grep -v -E "$(echo "$with_vals" | + awk -F "=" '{print $1}')" | awk '{print $0"="}') +else + without_vals=$(echo "$scripts" | tr '|' '\n' | awk '{print $0"="}') +fi + +if [ -n "$without_vals" ]; then + echo "$without_vals" +fi diff --git a/straper/db/public/zfs/etc-zfs/zpool.d/test_status b/straper/db/public/zfs/etc-zfs/zpool.d/test_status new file mode 100755 index 0000000..8ad3e10 --- /dev/null +++ b/straper/db/public/zfs/etc-zfs/zpool.d/test_status @@ -0,0 +1,249 @@ +#!/bin/sh +# +# Show SMART stats +# + +helpstr=" +smart: Show SMART temperature and error stats (specific to drive type) +smartx: Show SMART extended drive stats (specific to drive type). +temp: Show SMART drive temperature in celsius (all drives). +health: Show reported SMART status (all drives). +r_proc: Show SMART read GBytes processed over drive lifetime (SAS). +w_proc: Show SMART write GBytes processed over drive lifetime (SAS). +r_ucor: Show SMART read uncorrectable errors (SAS). +w_ucor: Show SMART write uncorrectable errors (SAS). +nonmed: Show SMART non-medium errors (SAS). +defect: Show SMART grown defect list (SAS). +hours_on: Show number of hours drive powered on (all drives). +realloc: Show SMART reallocated sectors count (ATA). +rep_ucor: Show SMART reported uncorrectable count (ATA). +cmd_to: Show SMART command timeout count (ATA). +pend_sec: Show SMART current pending sector count (ATA). +off_ucor: Show SMART offline uncorrectable errors (ATA). +ata_err: Show SMART ATA errors (ATA). +pwr_cyc: Show SMART power cycle count (ATA). +serial: Show disk serial number. +nvme_err: Show SMART NVMe errors (NVMe). +smart_test: Show SMART self-test results summary. +test_type: Show SMART self-test type (short, long... ). +test_status: Show SMART self-test status. +test_progress: Show SMART self-test percentage done. +test_ended: Show when the last SMART self-test ended (if supported). +" + +# Hack for developer testing +# +# If you set $samples to a directory containing smartctl output text files, +# we will use them instead of running smartctl on the vdevs. This can be +# useful if you want to test a bunch of different smartctl outputs. Also, if +# $samples is set, and additional 'file' column is added to the zpool output +# showing the filename. +samples= + +# get_filename_from_dir DIR +# +# Look in directory DIR and return a filename from it. The filename returned +# is chosen quasi-sequentially (based off our PID). This allows us to return +# a different filename every time this script is invoked (which we do for each +# vdev), without having to maintain state. +get_filename_from_dir() +{ + dir=$1 + pid="$$" + num_files=$(find "$dir" -maxdepth 1 -type f | wc -l) + mod=$((pid % num_files)) + i=0 + find "$dir" -type f -printf '%f\n' | while read -r file ; do + if [ "$mod" = "$i" ] ; then + echo "$file" + break + fi + i=$((i+1)) + done +} + +script="${0##*/}" + +if [ "$1" = "-h" ] ; then + echo "$helpstr" | grep "$script:" | tr -s '\t' | cut -f 2- + exit +fi + +# Sometimes, UPATH ends up /dev/(null). +# That should be corrected, but for now... +# shellcheck disable=SC2154 +if [ ! -b "$VDEV_UPATH" ]; then + somepath="${VDEV_PATH}" +else + somepath="${VDEV_UPATH}" +fi + +if [ -b "$somepath" ] && PATH="/usr/sbin:$PATH" command -v smartctl > /dev/null || [ -n "$samples" ] ; then + if [ -n "$samples" ] ; then + # cat a smartctl output text file instead of running smartctl + # on a vdev (only used for developer testing). + file=$(get_filename_from_dir "$samples") + echo "file=$file" + raw_out=$(cat "$samples/$file") + else + raw_out=$(sudo smartctl -a "$somepath") + fi + + # What kind of drive are we? Look for the right line in smartctl: + # + # SAS: + # Transport protocol: SAS + # + # SATA: + # ATA Version is: 8 + # + # NVMe: + # SMART/Health Information (NVMe Log 0xnn, NSID 0xnn) + # + out=$(echo "$raw_out" | awk ' +# SAS specific +/read:/{print "rrd="$4"\nr_cor="$5"\nr_proc="$7"\nr_ucor="$8} +/write:/{print "rwr="$4"\nw_cor="$5"\nw_proc="$7"\nw_ucor="$8} +/Non-medium error count/{print "nonmed="$4} +/Elements in grown defect list/{print "defect="$6} + +# SAS common +/SAS/{type="sas"} +/Drive Temperature:/{print "temp="$4} +# Status can be a long string, substitute spaces for '_' +/SMART Health Status:/{printf "health="; for(i=4;i<=NF-1;i++){printf "%s_", $i}; printf "%s\n", $i} +/number of hours powered up/{print "hours_on="$7; hours_on=int($7)} +/Serial number:/{print "serial="$3} + +# SATA specific +/Reallocated_Sector_Ct/{print "realloc="$10} +/Reported_Uncorrect/{print "rep_ucor="$10} +/Command_Timeout/{print "cmd_to="$10} +/Current_Pending_Sector/{print "pend_sec="$10} +/Offline_Uncorrectable/{print "off_ucor="$10} +/ATA Error Count:/{print "ata_err="$4} +/Power_Cycle_Count/{print "pwr_cyc="$10} + +# SATA common +/SATA/{type="sata"} +/Temperature_Celsius/{print "temp="$10} +/Airflow_Temperature_Cel/{print "temp="$10} +/Current Temperature:/{print "temp="$3} +/SMART overall-health self-assessment test result:/{print "health="$6} +/Power_On_Hours/{print "hours_on="$10; hours_on=int($10)} +/Serial Number:/{print "serial="$3} + +# NVMe common +/NVMe/{type="nvme"} +/Temperature:/{print "temp="$2} +/SMART overall-health self-assessment test result:/{print "health="$6} +/Power On Hours:/{gsub("[^0-9]","",$4); print "hours_on="$4} +/Serial Number:/{print "serial="$3} +/Power Cycles:/{print "pwr_cyc="$3} + +# NVMe specific +/Media and Data Integrity Errors:/{print "nvme_err="$6} + +# SMART self-test info +/Self-test execution status:/{progress=tolower($4)} # SAS +/SMART Self-test log/{test_seen=1} # SAS +/SMART Extended Self-test Log/{test_seen=1} # SATA +/# 1/{ + test_type=tolower($3"_"$4); + # Status could be one word ("Completed") or multiple ("Completed: read + # failure"). Look for the ":" to see if we need to grab more words. + + if ($5 ~ ":") + status=tolower($5""$6"_"$7) + else + status=tolower($5) + if (status=="self") + status="running"; + + if (type == "sas") { + hours=int($(NF-4)) + } else { + hours=int($(NF-1)) + # SATA reports percent remaining, rather than percent done + # Convert it to percent done. + progress=(100-int($(NF-2)))"%" + } + # When we int()-ify "hours", it converts stuff like "NOW" and "-" into + # 0. In those cases, set it to hours_on, so they will cancel out in + # the "hours_ago" calculation later on. + if (hours == 0) + hours=hours_on + + if (test_seen) { + print "test="hours_on + print "test_type="test_type + print "test_status="status + print "test_progress="progress + } + # Not all drives report hours_on + if (hours_on && hours) { + total_hours_ago=(hours_on-hours) + days_ago=int(total_hours_ago/24) + hours_ago=(total_hours_ago % 24) + if (days_ago != 0) + ago_str=days_ago"d" + if (hours_ago !=0) + ago_str=ago_str""hours_ago"h" + print "test_ended="ago_str + } +} + +END {print "type="type; ORS="\n"; print ""} +'); +fi +type=$(echo "$out" | grep '^type=' | cut -d '=' -f 2) + +# If type is not set by now, either we don't have a block device +# or smartctl failed. Either way, default to ATA and set $out to +# nothing. +if [ -z "$type" ]; then + type="sata" + out= +fi + +case $script in +smart) + # Print temperature plus common predictors of drive failure + if [ "$type" = "sas" ] ; then + scripts="temp|health|r_ucor|w_ucor" + elif [ "$type" = "sata" ] ; then + scripts="temp|health|ata_err|realloc|rep_ucor|cmd_to|pend_sec|off_ucor" + elif [ "$type" = "nvme" ] ; then + scripts="temp|health|nvme_err" + fi + ;; +smartx) + # Print some other interesting stats + if [ "$type" = "sas" ] ; then + scripts="hours_on|defect|nonmed|r_proc|w_proc" + elif [ "$type" = "sata" ] ; then + scripts="hours_on|pwr_cyc" + elif [ "$type" = "nvme" ] ; then + scripts="hours_on|pwr_cyc" + fi + ;; +smart_test) + scripts="test_type|test_status|test_progress|test_ended" + ;; +*) + scripts="$script" +esac + +with_vals=$(echo "$out" | grep -E "$scripts") +if [ -n "$with_vals" ]; then + echo "$with_vals" + without_vals=$(echo "$scripts" | tr '|' '\n' | + grep -v -E "$(echo "$with_vals" | + awk -F "=" '{print $1}')" | awk '{print $0"="}') +else + without_vals=$(echo "$scripts" | tr '|' '\n' | awk '{print $0"="}') +fi + +if [ -n "$without_vals" ]; then + echo "$without_vals" +fi diff --git a/straper/db/public/zfs/etc-zfs/zpool.d/test_type b/straper/db/public/zfs/etc-zfs/zpool.d/test_type new file mode 100755 index 0000000..8ad3e10 --- /dev/null +++ b/straper/db/public/zfs/etc-zfs/zpool.d/test_type @@ -0,0 +1,249 @@ +#!/bin/sh +# +# Show SMART stats +# + +helpstr=" +smart: Show SMART temperature and error stats (specific to drive type) +smartx: Show SMART extended drive stats (specific to drive type). +temp: Show SMART drive temperature in celsius (all drives). +health: Show reported SMART status (all drives). +r_proc: Show SMART read GBytes processed over drive lifetime (SAS). +w_proc: Show SMART write GBytes processed over drive lifetime (SAS). +r_ucor: Show SMART read uncorrectable errors (SAS). +w_ucor: Show SMART write uncorrectable errors (SAS). +nonmed: Show SMART non-medium errors (SAS). +defect: Show SMART grown defect list (SAS). +hours_on: Show number of hours drive powered on (all drives). +realloc: Show SMART reallocated sectors count (ATA). +rep_ucor: Show SMART reported uncorrectable count (ATA). +cmd_to: Show SMART command timeout count (ATA). +pend_sec: Show SMART current pending sector count (ATA). +off_ucor: Show SMART offline uncorrectable errors (ATA). +ata_err: Show SMART ATA errors (ATA). +pwr_cyc: Show SMART power cycle count (ATA). +serial: Show disk serial number. +nvme_err: Show SMART NVMe errors (NVMe). +smart_test: Show SMART self-test results summary. +test_type: Show SMART self-test type (short, long... ). +test_status: Show SMART self-test status. +test_progress: Show SMART self-test percentage done. +test_ended: Show when the last SMART self-test ended (if supported). +" + +# Hack for developer testing +# +# If you set $samples to a directory containing smartctl output text files, +# we will use them instead of running smartctl on the vdevs. This can be +# useful if you want to test a bunch of different smartctl outputs. Also, if +# $samples is set, and additional 'file' column is added to the zpool output +# showing the filename. +samples= + +# get_filename_from_dir DIR +# +# Look in directory DIR and return a filename from it. The filename returned +# is chosen quasi-sequentially (based off our PID). This allows us to return +# a different filename every time this script is invoked (which we do for each +# vdev), without having to maintain state. +get_filename_from_dir() +{ + dir=$1 + pid="$$" + num_files=$(find "$dir" -maxdepth 1 -type f | wc -l) + mod=$((pid % num_files)) + i=0 + find "$dir" -type f -printf '%f\n' | while read -r file ; do + if [ "$mod" = "$i" ] ; then + echo "$file" + break + fi + i=$((i+1)) + done +} + +script="${0##*/}" + +if [ "$1" = "-h" ] ; then + echo "$helpstr" | grep "$script:" | tr -s '\t' | cut -f 2- + exit +fi + +# Sometimes, UPATH ends up /dev/(null). +# That should be corrected, but for now... +# shellcheck disable=SC2154 +if [ ! -b "$VDEV_UPATH" ]; then + somepath="${VDEV_PATH}" +else + somepath="${VDEV_UPATH}" +fi + +if [ -b "$somepath" ] && PATH="/usr/sbin:$PATH" command -v smartctl > /dev/null || [ -n "$samples" ] ; then + if [ -n "$samples" ] ; then + # cat a smartctl output text file instead of running smartctl + # on a vdev (only used for developer testing). + file=$(get_filename_from_dir "$samples") + echo "file=$file" + raw_out=$(cat "$samples/$file") + else + raw_out=$(sudo smartctl -a "$somepath") + fi + + # What kind of drive are we? Look for the right line in smartctl: + # + # SAS: + # Transport protocol: SAS + # + # SATA: + # ATA Version is: 8 + # + # NVMe: + # SMART/Health Information (NVMe Log 0xnn, NSID 0xnn) + # + out=$(echo "$raw_out" | awk ' +# SAS specific +/read:/{print "rrd="$4"\nr_cor="$5"\nr_proc="$7"\nr_ucor="$8} +/write:/{print "rwr="$4"\nw_cor="$5"\nw_proc="$7"\nw_ucor="$8} +/Non-medium error count/{print "nonmed="$4} +/Elements in grown defect list/{print "defect="$6} + +# SAS common +/SAS/{type="sas"} +/Drive Temperature:/{print "temp="$4} +# Status can be a long string, substitute spaces for '_' +/SMART Health Status:/{printf "health="; for(i=4;i<=NF-1;i++){printf "%s_", $i}; printf "%s\n", $i} +/number of hours powered up/{print "hours_on="$7; hours_on=int($7)} +/Serial number:/{print "serial="$3} + +# SATA specific +/Reallocated_Sector_Ct/{print "realloc="$10} +/Reported_Uncorrect/{print "rep_ucor="$10} +/Command_Timeout/{print "cmd_to="$10} +/Current_Pending_Sector/{print "pend_sec="$10} +/Offline_Uncorrectable/{print "off_ucor="$10} +/ATA Error Count:/{print "ata_err="$4} +/Power_Cycle_Count/{print "pwr_cyc="$10} + +# SATA common +/SATA/{type="sata"} +/Temperature_Celsius/{print "temp="$10} +/Airflow_Temperature_Cel/{print "temp="$10} +/Current Temperature:/{print "temp="$3} +/SMART overall-health self-assessment test result:/{print "health="$6} +/Power_On_Hours/{print "hours_on="$10; hours_on=int($10)} +/Serial Number:/{print "serial="$3} + +# NVMe common +/NVMe/{type="nvme"} +/Temperature:/{print "temp="$2} +/SMART overall-health self-assessment test result:/{print "health="$6} +/Power On Hours:/{gsub("[^0-9]","",$4); print "hours_on="$4} +/Serial Number:/{print "serial="$3} +/Power Cycles:/{print "pwr_cyc="$3} + +# NVMe specific +/Media and Data Integrity Errors:/{print "nvme_err="$6} + +# SMART self-test info +/Self-test execution status:/{progress=tolower($4)} # SAS +/SMART Self-test log/{test_seen=1} # SAS +/SMART Extended Self-test Log/{test_seen=1} # SATA +/# 1/{ + test_type=tolower($3"_"$4); + # Status could be one word ("Completed") or multiple ("Completed: read + # failure"). Look for the ":" to see if we need to grab more words. + + if ($5 ~ ":") + status=tolower($5""$6"_"$7) + else + status=tolower($5) + if (status=="self") + status="running"; + + if (type == "sas") { + hours=int($(NF-4)) + } else { + hours=int($(NF-1)) + # SATA reports percent remaining, rather than percent done + # Convert it to percent done. + progress=(100-int($(NF-2)))"%" + } + # When we int()-ify "hours", it converts stuff like "NOW" and "-" into + # 0. In those cases, set it to hours_on, so they will cancel out in + # the "hours_ago" calculation later on. + if (hours == 0) + hours=hours_on + + if (test_seen) { + print "test="hours_on + print "test_type="test_type + print "test_status="status + print "test_progress="progress + } + # Not all drives report hours_on + if (hours_on && hours) { + total_hours_ago=(hours_on-hours) + days_ago=int(total_hours_ago/24) + hours_ago=(total_hours_ago % 24) + if (days_ago != 0) + ago_str=days_ago"d" + if (hours_ago !=0) + ago_str=ago_str""hours_ago"h" + print "test_ended="ago_str + } +} + +END {print "type="type; ORS="\n"; print ""} +'); +fi +type=$(echo "$out" | grep '^type=' | cut -d '=' -f 2) + +# If type is not set by now, either we don't have a block device +# or smartctl failed. Either way, default to ATA and set $out to +# nothing. +if [ -z "$type" ]; then + type="sata" + out= +fi + +case $script in +smart) + # Print temperature plus common predictors of drive failure + if [ "$type" = "sas" ] ; then + scripts="temp|health|r_ucor|w_ucor" + elif [ "$type" = "sata" ] ; then + scripts="temp|health|ata_err|realloc|rep_ucor|cmd_to|pend_sec|off_ucor" + elif [ "$type" = "nvme" ] ; then + scripts="temp|health|nvme_err" + fi + ;; +smartx) + # Print some other interesting stats + if [ "$type" = "sas" ] ; then + scripts="hours_on|defect|nonmed|r_proc|w_proc" + elif [ "$type" = "sata" ] ; then + scripts="hours_on|pwr_cyc" + elif [ "$type" = "nvme" ] ; then + scripts="hours_on|pwr_cyc" + fi + ;; +smart_test) + scripts="test_type|test_status|test_progress|test_ended" + ;; +*) + scripts="$script" +esac + +with_vals=$(echo "$out" | grep -E "$scripts") +if [ -n "$with_vals" ]; then + echo "$with_vals" + without_vals=$(echo "$scripts" | tr '|' '\n' | + grep -v -E "$(echo "$with_vals" | + awk -F "=" '{print $1}')" | awk '{print $0"="}') +else + without_vals=$(echo "$scripts" | tr '|' '\n' | awk '{print $0"="}') +fi + +if [ -n "$without_vals" ]; then + echo "$without_vals" +fi diff --git a/straper/db/public/zfs/etc-zfs/zpool.d/upath b/straper/db/public/zfs/etc-zfs/zpool.d/upath new file mode 100755 index 0000000..e37ee1b --- /dev/null +++ b/straper/db/public/zfs/etc-zfs/zpool.d/upath @@ -0,0 +1,8 @@ +#!/bin/sh +if [ "$1" = "-h" ] ; then + echo "Show the underlying path for a device." + exit +fi + +# shellcheck disable=SC2154 +echo upath="$VDEV_UPATH" diff --git a/straper/db/public/zfs/etc-zfs/zpool.d/vendor b/straper/db/public/zfs/etc-zfs/zpool.d/vendor new file mode 100755 index 0000000..293effd --- /dev/null +++ b/straper/db/public/zfs/etc-zfs/zpool.d/vendor @@ -0,0 +1,92 @@ +#!/bin/sh +# +# Print some common lsblk values +# +# Any (lowercased) name symlinked to the lsblk script will be passed to lsblk +# as one of its --output names. Here's a partial list of --output names +# from the lsblk binary: +# +# Available columns (for --output): +# NAME device name +# KNAME internal kernel device name +# MAJ:MIN major:minor device number +# FSTYPE filesystem type +# MOUNTPOINT where the device is mounted +# LABEL filesystem LABEL +# UUID filesystem UUID +# RA read-ahead of the device +# RO read-only device +# RM removable device +# MODEL device identifier +# SIZE size of the device +# STATE state of the device +# OWNER user name +# GROUP group name +# MODE device node permissions +# ALIGNMENT alignment offset +# MIN-IO minimum I/O size +# OPT-IO optimal I/O size +# PHY-SEC physical sector size +# LOG-SEC logical sector size +# ROTA rotational device +# SCHED I/O scheduler name +# RQ-SIZE request queue size +# TYPE device type +# DISC-ALN discard alignment offset +# DISC-GRAN discard granularity +# DISC-MAX discard max bytes +# DISC-ZERO discard zeroes data +# +# If the script is run as just 'lsblk' then print out disk size, vendor, +# and model number. + + +helpstr=" +label: Show filesystem label. +model: Show disk model number. +size: Show the disk capacity. +vendor: Show the disk vendor. +lsblk: Show the disk size, vendor, and model number." + +script="${0##*/}" + +if [ "$1" = "-h" ] ; then + echo "$helpstr" | grep "$script:" | tr -s '\t' | cut -f 2- + exit +fi + +if [ "$script" = "lsblk" ] ; then + list="size vendor model" +else + list=$(echo "$script" | tr '[:upper:]' '[:lower:]') +fi + +# Sometimes, UPATH ends up /dev/(null). +# That should be corrected, but for now... +# shellcheck disable=SC2154 +if [ ! -b "$VDEV_UPATH" ]; then + somepath="${VDEV_PATH}" +else + somepath="${VDEV_UPATH}" +fi + +# Older versions of lsblk don't support all these values (like SERIAL). +for i in $list ; do + + # Special case: Looking up the size of a file-based vdev can't + # be done with lsblk. + if [ "$i" = "size" ] && [ -f "$somepath" ] ; then + size=$(du -h --apparent-size "$somepath" | cut -f 1) + echo "size=$size" + continue + fi + + + val="" + if val=$(eval "lsblk -dl -n -o $i $somepath 2>/dev/null") ; then + # Remove leading/trailing whitespace from value + val=$(echo "$val" | sed -e 's/^[[:space:]]*//' \ + -e 's/[[:space:]]*$//') + fi + echo "$i=$val" +done diff --git a/straper/db/public/zfs/etc-zfs/zpool.d/w_proc b/straper/db/public/zfs/etc-zfs/zpool.d/w_proc new file mode 100755 index 0000000..8ad3e10 --- /dev/null +++ b/straper/db/public/zfs/etc-zfs/zpool.d/w_proc @@ -0,0 +1,249 @@ +#!/bin/sh +# +# Show SMART stats +# + +helpstr=" +smart: Show SMART temperature and error stats (specific to drive type) +smartx: Show SMART extended drive stats (specific to drive type). +temp: Show SMART drive temperature in celsius (all drives). +health: Show reported SMART status (all drives). +r_proc: Show SMART read GBytes processed over drive lifetime (SAS). +w_proc: Show SMART write GBytes processed over drive lifetime (SAS). +r_ucor: Show SMART read uncorrectable errors (SAS). +w_ucor: Show SMART write uncorrectable errors (SAS). +nonmed: Show SMART non-medium errors (SAS). +defect: Show SMART grown defect list (SAS). +hours_on: Show number of hours drive powered on (all drives). +realloc: Show SMART reallocated sectors count (ATA). +rep_ucor: Show SMART reported uncorrectable count (ATA). +cmd_to: Show SMART command timeout count (ATA). +pend_sec: Show SMART current pending sector count (ATA). +off_ucor: Show SMART offline uncorrectable errors (ATA). +ata_err: Show SMART ATA errors (ATA). +pwr_cyc: Show SMART power cycle count (ATA). +serial: Show disk serial number. +nvme_err: Show SMART NVMe errors (NVMe). +smart_test: Show SMART self-test results summary. +test_type: Show SMART self-test type (short, long... ). +test_status: Show SMART self-test status. +test_progress: Show SMART self-test percentage done. +test_ended: Show when the last SMART self-test ended (if supported). +" + +# Hack for developer testing +# +# If you set $samples to a directory containing smartctl output text files, +# we will use them instead of running smartctl on the vdevs. This can be +# useful if you want to test a bunch of different smartctl outputs. Also, if +# $samples is set, and additional 'file' column is added to the zpool output +# showing the filename. +samples= + +# get_filename_from_dir DIR +# +# Look in directory DIR and return a filename from it. The filename returned +# is chosen quasi-sequentially (based off our PID). This allows us to return +# a different filename every time this script is invoked (which we do for each +# vdev), without having to maintain state. +get_filename_from_dir() +{ + dir=$1 + pid="$$" + num_files=$(find "$dir" -maxdepth 1 -type f | wc -l) + mod=$((pid % num_files)) + i=0 + find "$dir" -type f -printf '%f\n' | while read -r file ; do + if [ "$mod" = "$i" ] ; then + echo "$file" + break + fi + i=$((i+1)) + done +} + +script="${0##*/}" + +if [ "$1" = "-h" ] ; then + echo "$helpstr" | grep "$script:" | tr -s '\t' | cut -f 2- + exit +fi + +# Sometimes, UPATH ends up /dev/(null). +# That should be corrected, but for now... +# shellcheck disable=SC2154 +if [ ! -b "$VDEV_UPATH" ]; then + somepath="${VDEV_PATH}" +else + somepath="${VDEV_UPATH}" +fi + +if [ -b "$somepath" ] && PATH="/usr/sbin:$PATH" command -v smartctl > /dev/null || [ -n "$samples" ] ; then + if [ -n "$samples" ] ; then + # cat a smartctl output text file instead of running smartctl + # on a vdev (only used for developer testing). + file=$(get_filename_from_dir "$samples") + echo "file=$file" + raw_out=$(cat "$samples/$file") + else + raw_out=$(sudo smartctl -a "$somepath") + fi + + # What kind of drive are we? Look for the right line in smartctl: + # + # SAS: + # Transport protocol: SAS + # + # SATA: + # ATA Version is: 8 + # + # NVMe: + # SMART/Health Information (NVMe Log 0xnn, NSID 0xnn) + # + out=$(echo "$raw_out" | awk ' +# SAS specific +/read:/{print "rrd="$4"\nr_cor="$5"\nr_proc="$7"\nr_ucor="$8} +/write:/{print "rwr="$4"\nw_cor="$5"\nw_proc="$7"\nw_ucor="$8} +/Non-medium error count/{print "nonmed="$4} +/Elements in grown defect list/{print "defect="$6} + +# SAS common +/SAS/{type="sas"} +/Drive Temperature:/{print "temp="$4} +# Status can be a long string, substitute spaces for '_' +/SMART Health Status:/{printf "health="; for(i=4;i<=NF-1;i++){printf "%s_", $i}; printf "%s\n", $i} +/number of hours powered up/{print "hours_on="$7; hours_on=int($7)} +/Serial number:/{print "serial="$3} + +# SATA specific +/Reallocated_Sector_Ct/{print "realloc="$10} +/Reported_Uncorrect/{print "rep_ucor="$10} +/Command_Timeout/{print "cmd_to="$10} +/Current_Pending_Sector/{print "pend_sec="$10} +/Offline_Uncorrectable/{print "off_ucor="$10} +/ATA Error Count:/{print "ata_err="$4} +/Power_Cycle_Count/{print "pwr_cyc="$10} + +# SATA common +/SATA/{type="sata"} +/Temperature_Celsius/{print "temp="$10} +/Airflow_Temperature_Cel/{print "temp="$10} +/Current Temperature:/{print "temp="$3} +/SMART overall-health self-assessment test result:/{print "health="$6} +/Power_On_Hours/{print "hours_on="$10; hours_on=int($10)} +/Serial Number:/{print "serial="$3} + +# NVMe common +/NVMe/{type="nvme"} +/Temperature:/{print "temp="$2} +/SMART overall-health self-assessment test result:/{print "health="$6} +/Power On Hours:/{gsub("[^0-9]","",$4); print "hours_on="$4} +/Serial Number:/{print "serial="$3} +/Power Cycles:/{print "pwr_cyc="$3} + +# NVMe specific +/Media and Data Integrity Errors:/{print "nvme_err="$6} + +# SMART self-test info +/Self-test execution status:/{progress=tolower($4)} # SAS +/SMART Self-test log/{test_seen=1} # SAS +/SMART Extended Self-test Log/{test_seen=1} # SATA +/# 1/{ + test_type=tolower($3"_"$4); + # Status could be one word ("Completed") or multiple ("Completed: read + # failure"). Look for the ":" to see if we need to grab more words. + + if ($5 ~ ":") + status=tolower($5""$6"_"$7) + else + status=tolower($5) + if (status=="self") + status="running"; + + if (type == "sas") { + hours=int($(NF-4)) + } else { + hours=int($(NF-1)) + # SATA reports percent remaining, rather than percent done + # Convert it to percent done. + progress=(100-int($(NF-2)))"%" + } + # When we int()-ify "hours", it converts stuff like "NOW" and "-" into + # 0. In those cases, set it to hours_on, so they will cancel out in + # the "hours_ago" calculation later on. + if (hours == 0) + hours=hours_on + + if (test_seen) { + print "test="hours_on + print "test_type="test_type + print "test_status="status + print "test_progress="progress + } + # Not all drives report hours_on + if (hours_on && hours) { + total_hours_ago=(hours_on-hours) + days_ago=int(total_hours_ago/24) + hours_ago=(total_hours_ago % 24) + if (days_ago != 0) + ago_str=days_ago"d" + if (hours_ago !=0) + ago_str=ago_str""hours_ago"h" + print "test_ended="ago_str + } +} + +END {print "type="type; ORS="\n"; print ""} +'); +fi +type=$(echo "$out" | grep '^type=' | cut -d '=' -f 2) + +# If type is not set by now, either we don't have a block device +# or smartctl failed. Either way, default to ATA and set $out to +# nothing. +if [ -z "$type" ]; then + type="sata" + out= +fi + +case $script in +smart) + # Print temperature plus common predictors of drive failure + if [ "$type" = "sas" ] ; then + scripts="temp|health|r_ucor|w_ucor" + elif [ "$type" = "sata" ] ; then + scripts="temp|health|ata_err|realloc|rep_ucor|cmd_to|pend_sec|off_ucor" + elif [ "$type" = "nvme" ] ; then + scripts="temp|health|nvme_err" + fi + ;; +smartx) + # Print some other interesting stats + if [ "$type" = "sas" ] ; then + scripts="hours_on|defect|nonmed|r_proc|w_proc" + elif [ "$type" = "sata" ] ; then + scripts="hours_on|pwr_cyc" + elif [ "$type" = "nvme" ] ; then + scripts="hours_on|pwr_cyc" + fi + ;; +smart_test) + scripts="test_type|test_status|test_progress|test_ended" + ;; +*) + scripts="$script" +esac + +with_vals=$(echo "$out" | grep -E "$scripts") +if [ -n "$with_vals" ]; then + echo "$with_vals" + without_vals=$(echo "$scripts" | tr '|' '\n' | + grep -v -E "$(echo "$with_vals" | + awk -F "=" '{print $1}')" | awk '{print $0"="}') +else + without_vals=$(echo "$scripts" | tr '|' '\n' | awk '{print $0"="}') +fi + +if [ -n "$without_vals" ]; then + echo "$without_vals" +fi diff --git a/straper/db/public/zfs/etc-zfs/zpool.d/w_ucor b/straper/db/public/zfs/etc-zfs/zpool.d/w_ucor new file mode 100755 index 0000000..8ad3e10 --- /dev/null +++ b/straper/db/public/zfs/etc-zfs/zpool.d/w_ucor @@ -0,0 +1,249 @@ +#!/bin/sh +# +# Show SMART stats +# + +helpstr=" +smart: Show SMART temperature and error stats (specific to drive type) +smartx: Show SMART extended drive stats (specific to drive type). +temp: Show SMART drive temperature in celsius (all drives). +health: Show reported SMART status (all drives). +r_proc: Show SMART read GBytes processed over drive lifetime (SAS). +w_proc: Show SMART write GBytes processed over drive lifetime (SAS). +r_ucor: Show SMART read uncorrectable errors (SAS). +w_ucor: Show SMART write uncorrectable errors (SAS). +nonmed: Show SMART non-medium errors (SAS). +defect: Show SMART grown defect list (SAS). +hours_on: Show number of hours drive powered on (all drives). +realloc: Show SMART reallocated sectors count (ATA). +rep_ucor: Show SMART reported uncorrectable count (ATA). +cmd_to: Show SMART command timeout count (ATA). +pend_sec: Show SMART current pending sector count (ATA). +off_ucor: Show SMART offline uncorrectable errors (ATA). +ata_err: Show SMART ATA errors (ATA). +pwr_cyc: Show SMART power cycle count (ATA). +serial: Show disk serial number. +nvme_err: Show SMART NVMe errors (NVMe). +smart_test: Show SMART self-test results summary. +test_type: Show SMART self-test type (short, long... ). +test_status: Show SMART self-test status. +test_progress: Show SMART self-test percentage done. +test_ended: Show when the last SMART self-test ended (if supported). +" + +# Hack for developer testing +# +# If you set $samples to a directory containing smartctl output text files, +# we will use them instead of running smartctl on the vdevs. This can be +# useful if you want to test a bunch of different smartctl outputs. Also, if +# $samples is set, and additional 'file' column is added to the zpool output +# showing the filename. +samples= + +# get_filename_from_dir DIR +# +# Look in directory DIR and return a filename from it. The filename returned +# is chosen quasi-sequentially (based off our PID). This allows us to return +# a different filename every time this script is invoked (which we do for each +# vdev), without having to maintain state. +get_filename_from_dir() +{ + dir=$1 + pid="$$" + num_files=$(find "$dir" -maxdepth 1 -type f | wc -l) + mod=$((pid % num_files)) + i=0 + find "$dir" -type f -printf '%f\n' | while read -r file ; do + if [ "$mod" = "$i" ] ; then + echo "$file" + break + fi + i=$((i+1)) + done +} + +script="${0##*/}" + +if [ "$1" = "-h" ] ; then + echo "$helpstr" | grep "$script:" | tr -s '\t' | cut -f 2- + exit +fi + +# Sometimes, UPATH ends up /dev/(null). +# That should be corrected, but for now... +# shellcheck disable=SC2154 +if [ ! -b "$VDEV_UPATH" ]; then + somepath="${VDEV_PATH}" +else + somepath="${VDEV_UPATH}" +fi + +if [ -b "$somepath" ] && PATH="/usr/sbin:$PATH" command -v smartctl > /dev/null || [ -n "$samples" ] ; then + if [ -n "$samples" ] ; then + # cat a smartctl output text file instead of running smartctl + # on a vdev (only used for developer testing). + file=$(get_filename_from_dir "$samples") + echo "file=$file" + raw_out=$(cat "$samples/$file") + else + raw_out=$(sudo smartctl -a "$somepath") + fi + + # What kind of drive are we? Look for the right line in smartctl: + # + # SAS: + # Transport protocol: SAS + # + # SATA: + # ATA Version is: 8 + # + # NVMe: + # SMART/Health Information (NVMe Log 0xnn, NSID 0xnn) + # + out=$(echo "$raw_out" | awk ' +# SAS specific +/read:/{print "rrd="$4"\nr_cor="$5"\nr_proc="$7"\nr_ucor="$8} +/write:/{print "rwr="$4"\nw_cor="$5"\nw_proc="$7"\nw_ucor="$8} +/Non-medium error count/{print "nonmed="$4} +/Elements in grown defect list/{print "defect="$6} + +# SAS common +/SAS/{type="sas"} +/Drive Temperature:/{print "temp="$4} +# Status can be a long string, substitute spaces for '_' +/SMART Health Status:/{printf "health="; for(i=4;i<=NF-1;i++){printf "%s_", $i}; printf "%s\n", $i} +/number of hours powered up/{print "hours_on="$7; hours_on=int($7)} +/Serial number:/{print "serial="$3} + +# SATA specific +/Reallocated_Sector_Ct/{print "realloc="$10} +/Reported_Uncorrect/{print "rep_ucor="$10} +/Command_Timeout/{print "cmd_to="$10} +/Current_Pending_Sector/{print "pend_sec="$10} +/Offline_Uncorrectable/{print "off_ucor="$10} +/ATA Error Count:/{print "ata_err="$4} +/Power_Cycle_Count/{print "pwr_cyc="$10} + +# SATA common +/SATA/{type="sata"} +/Temperature_Celsius/{print "temp="$10} +/Airflow_Temperature_Cel/{print "temp="$10} +/Current Temperature:/{print "temp="$3} +/SMART overall-health self-assessment test result:/{print "health="$6} +/Power_On_Hours/{print "hours_on="$10; hours_on=int($10)} +/Serial Number:/{print "serial="$3} + +# NVMe common +/NVMe/{type="nvme"} +/Temperature:/{print "temp="$2} +/SMART overall-health self-assessment test result:/{print "health="$6} +/Power On Hours:/{gsub("[^0-9]","",$4); print "hours_on="$4} +/Serial Number:/{print "serial="$3} +/Power Cycles:/{print "pwr_cyc="$3} + +# NVMe specific +/Media and Data Integrity Errors:/{print "nvme_err="$6} + +# SMART self-test info +/Self-test execution status:/{progress=tolower($4)} # SAS +/SMART Self-test log/{test_seen=1} # SAS +/SMART Extended Self-test Log/{test_seen=1} # SATA +/# 1/{ + test_type=tolower($3"_"$4); + # Status could be one word ("Completed") or multiple ("Completed: read + # failure"). Look for the ":" to see if we need to grab more words. + + if ($5 ~ ":") + status=tolower($5""$6"_"$7) + else + status=tolower($5) + if (status=="self") + status="running"; + + if (type == "sas") { + hours=int($(NF-4)) + } else { + hours=int($(NF-1)) + # SATA reports percent remaining, rather than percent done + # Convert it to percent done. + progress=(100-int($(NF-2)))"%" + } + # When we int()-ify "hours", it converts stuff like "NOW" and "-" into + # 0. In those cases, set it to hours_on, so they will cancel out in + # the "hours_ago" calculation later on. + if (hours == 0) + hours=hours_on + + if (test_seen) { + print "test="hours_on + print "test_type="test_type + print "test_status="status + print "test_progress="progress + } + # Not all drives report hours_on + if (hours_on && hours) { + total_hours_ago=(hours_on-hours) + days_ago=int(total_hours_ago/24) + hours_ago=(total_hours_ago % 24) + if (days_ago != 0) + ago_str=days_ago"d" + if (hours_ago !=0) + ago_str=ago_str""hours_ago"h" + print "test_ended="ago_str + } +} + +END {print "type="type; ORS="\n"; print ""} +'); +fi +type=$(echo "$out" | grep '^type=' | cut -d '=' -f 2) + +# If type is not set by now, either we don't have a block device +# or smartctl failed. Either way, default to ATA and set $out to +# nothing. +if [ -z "$type" ]; then + type="sata" + out= +fi + +case $script in +smart) + # Print temperature plus common predictors of drive failure + if [ "$type" = "sas" ] ; then + scripts="temp|health|r_ucor|w_ucor" + elif [ "$type" = "sata" ] ; then + scripts="temp|health|ata_err|realloc|rep_ucor|cmd_to|pend_sec|off_ucor" + elif [ "$type" = "nvme" ] ; then + scripts="temp|health|nvme_err" + fi + ;; +smartx) + # Print some other interesting stats + if [ "$type" = "sas" ] ; then + scripts="hours_on|defect|nonmed|r_proc|w_proc" + elif [ "$type" = "sata" ] ; then + scripts="hours_on|pwr_cyc" + elif [ "$type" = "nvme" ] ; then + scripts="hours_on|pwr_cyc" + fi + ;; +smart_test) + scripts="test_type|test_status|test_progress|test_ended" + ;; +*) + scripts="$script" +esac + +with_vals=$(echo "$out" | grep -E "$scripts") +if [ -n "$with_vals" ]; then + echo "$with_vals" + without_vals=$(echo "$scripts" | tr '|' '\n' | + grep -v -E "$(echo "$with_vals" | + awk -F "=" '{print $1}')" | awk '{print $0"="}') +else + without_vals=$(echo "$scripts" | tr '|' '\n' | awk '{print $0"="}') +fi + +if [ -n "$without_vals" ]; then + echo "$without_vals" +fi -- cgit v1.3