diff options
Diffstat (limited to 'straper/db/public/zfs/etc-zfs')
59 files changed, 10384 insertions, 0 deletions
diff --git a/straper/db/public/zfs/etc-zfs/zed.d/all-syslog.sh b/straper/db/public/zfs/etc-zfs/zed.d/all-syslog.sh new file mode 100755 index 0000000..5f60114 --- /dev/null +++ b/straper/db/public/zfs/etc-zfs/zed.d/all-syslog.sh @@ -0,0 +1,52 @@ +#!/bin/sh +# shellcheck disable=SC2154 +# +# Copyright (C) 2013-2014 Lawrence Livermore National Security, LLC. +# Copyright (c) 2020 by Delphix. All rights reserved. +# + +# +# Log the zevent via syslog. +# + +[ -f "${ZED_ZEDLET_DIR}/zed.rc" ] && . "${ZED_ZEDLET_DIR}/zed.rc" +. "${ZED_ZEDLET_DIR}/zed-functions.sh" + +zed_exit_if_ignoring_this_event + +# build a string of name=value pairs for this event +msg="eid=${ZEVENT_EID} class=${ZEVENT_SUBCLASS}" + +if [ "${ZED_SYSLOG_DISPLAY_GUIDS}" = "1" ]; then + [ -n "${ZEVENT_POOL_GUID}" ] && msg="${msg} pool_guid=${ZEVENT_POOL_GUID}" + [ -n "${ZEVENT_VDEV_GUID}" ] && msg="${msg} vdev_guid=${ZEVENT_VDEV_GUID}" +else + [ -n "${ZEVENT_POOL}" ] && msg="${msg} pool='${ZEVENT_POOL}'" + [ -n "${ZEVENT_VDEV_PATH}" ] && msg="${msg} vdev=${ZEVENT_VDEV_PATH##*/}" +fi + +# log pool state if state is anything other than 'ACTIVE' +[ -n "${ZEVENT_POOL_STATE_STR}" ] && [ "$ZEVENT_POOL_STATE" -ne 0 ] && \ + msg="${msg} pool_state=${ZEVENT_POOL_STATE_STR}" + +# Log the following payload nvpairs if they are present +[ -n "${ZEVENT_VDEV_STATE_STR}" ] && msg="${msg} vdev_state=${ZEVENT_VDEV_STATE_STR}" +[ -n "${ZEVENT_CKSUM_ALGORITHM}" ] && msg="${msg} algorithm=${ZEVENT_CKSUM_ALGORITHM}" +[ -n "${ZEVENT_ZIO_SIZE}" ] && msg="${msg} size=${ZEVENT_ZIO_SIZE}" +[ -n "${ZEVENT_ZIO_OFFSET}" ] && msg="${msg} offset=${ZEVENT_ZIO_OFFSET}" +[ -n "${ZEVENT_ZIO_PRIORITY}" ] && msg="${msg} priority=${ZEVENT_ZIO_PRIORITY}" +[ -n "${ZEVENT_ZIO_ERR}" ] && msg="${msg} err=${ZEVENT_ZIO_ERR}" +[ -n "${ZEVENT_ZIO_FLAGS}" ] && msg="${msg} flags=$(printf '0x%x' "${ZEVENT_ZIO_FLAGS}")" + +# log delays that are >= 10 milisec +[ -n "${ZEVENT_ZIO_DELAY}" ] && [ "$ZEVENT_ZIO_DELAY" -gt 10000000 ] && \ + msg="${msg} delay=$((ZEVENT_ZIO_DELAY / 1000000))ms" + +# list the bookmark data together +# shellcheck disable=SC2153 +[ -n "${ZEVENT_ZIO_OBJSET}" ] && \ + msg="${msg} bookmark=${ZEVENT_ZIO_OBJSET}:${ZEVENT_ZIO_OBJECT}:${ZEVENT_ZIO_LEVEL}:${ZEVENT_ZIO_BLKID}" + +zed_log_msg "${msg}" + +exit 0 diff --git a/straper/db/public/zfs/etc-zfs/zed.d/data-notify.sh b/straper/db/public/zfs/etc-zfs/zed.d/data-notify.sh new file mode 100755 index 0000000..9846769 --- /dev/null +++ b/straper/db/public/zfs/etc-zfs/zed.d/data-notify.sh @@ -0,0 +1,44 @@ +#!/bin/sh +# shellcheck disable=SC2154 +# +# Send notification in response to a DATA error. +# +# Only one notification per ZED_NOTIFY_INTERVAL_SECS will be sent for a given +# class/pool/[vdev] combination. This protects against spamming the recipient +# should multiple events occur together in time for the same pool/[vdev]. +# +# Exit codes: +# 0: notification sent +# 1: notification failed +# 2: notification not configured +# 3: notification suppressed +# 9: internal error + +[ -f "${ZED_ZEDLET_DIR}/zed.rc" ] && . "${ZED_ZEDLET_DIR}/zed.rc" +. "${ZED_ZEDLET_DIR}/zed-functions.sh" + +[ -n "${ZEVENT_POOL}" ] || exit 9 +[ -n "${ZEVENT_SUBCLASS}" ] || exit 9 +[ -n "${ZED_NOTIFY_DATA}" ] || exit 3 + +rate_limit_tag="${ZEVENT_POOL};${ZEVENT_VDEV_GUID:-0};${ZEVENT_SUBCLASS};notify" +zed_rate_limit "${rate_limit_tag}" || exit 3 + +umask 077 +note_subject="ZFS ${ZEVENT_SUBCLASS} error for ${ZEVENT_POOL} on $(hostname)" +note_pathname="$(mktemp)" +{ + echo "ZFS has detected a data error:" + echo + echo " eid: ${ZEVENT_EID}" + echo " class: ${ZEVENT_SUBCLASS}" + echo " host: $(hostname)" + echo " time: ${ZEVENT_TIME_STRING}" + echo " error: ${ZEVENT_ZIO_ERR}" + echo " objid: ${ZEVENT_ZIO_OBJSET}:${ZEVENT_ZIO_OBJECT}" + echo " pool: ${ZEVENT_POOL}" +} > "${note_pathname}" + +zed_notify "${note_subject}" "${note_pathname}"; rv=$? +rm -f "${note_pathname}" +exit "${rv}" diff --git a/straper/db/public/zfs/etc-zfs/zed.d/deadman-slot_off.sh b/straper/db/public/zfs/etc-zfs/zed.d/deadman-slot_off.sh new file mode 100755 index 0000000..7b339b3 --- /dev/null +++ b/straper/db/public/zfs/etc-zfs/zed.d/deadman-slot_off.sh @@ -0,0 +1,71 @@ +#!/bin/sh +# shellcheck disable=SC3014,SC2154,SC2086,SC2034 +# +# Turn off disk's enclosure slot if an I/O is hung triggering the deadman. +# +# It's possible for outstanding I/O to a misbehaving SCSI disk to neither +# promptly complete or return an error. This can occur due to retry and +# recovery actions taken by the SCSI layer, driver, or disk. When it occurs +# the pool will be unresponsive even though there may be sufficient redundancy +# configured to proceeded without this single disk. +# +# When a hung I/O is detected by the kmods it will be posted as a deadman +# event. By default an I/O is considered to be hung after 5 minutes. This +# value can be changed with the zfs_deadman_ziotime_ms module parameter. +# If ZED_POWER_OFF_ENCLOSURE_SLOT_ON_DEADMAN is set the disk's enclosure +# slot will be powered off causing the outstanding I/O to fail. The ZED +# will then handle this like a normal disk failure and FAULT the vdev. +# +# We assume the user will be responsible for turning the slot back on +# after replacing the disk. +# +# Note that this script requires that your enclosure be supported by the +# Linux SCSI Enclosure services (SES) driver. The script will do nothing +# if you have no enclosure, or if your enclosure isn't supported. +# +# Exit codes: +# 0: slot successfully powered off +# 1: enclosure not available +# 2: ZED_POWER_OFF_ENCLOSURE_SLOT_ON_DEADMAN disabled +# 3: System not configured to wait on deadman +# 4: The enclosure sysfs path passed from ZFS does not exist +# 5: Enclosure slot didn't actually turn off after we told it to + +[ -f "${ZED_ZEDLET_DIR}/zed.rc" ] && . "${ZED_ZEDLET_DIR}/zed.rc" +. "${ZED_ZEDLET_DIR}/zed-functions.sh" + +if [ ! -d /sys/class/enclosure ] ; then + # No JBOD enclosure or NVMe slots + exit 1 +fi + +if [ "${ZED_POWER_OFF_ENCLOSURE_SLOT_ON_DEADMAN}" != "1" ] ; then + exit 2 +fi + +if [ "$ZEVENT_POOL_FAILMODE" != "wait" ] ; then + exit 3 +fi + +if [ ! -f "$ZEVENT_VDEV_ENC_SYSFS_PATH/power_status" ] ; then + exit 4 +fi + +# Turn off the slot and wait for sysfs to report that the slot is off. +# It can take ~400ms on some enclosures and multiple retries may be needed. +for i in $(seq 1 20) ; do + echo "off" | tee "$ZEVENT_VDEV_ENC_SYSFS_PATH/power_status" + + for j in $(seq 1 5) ; do + if [ "$(cat $ZEVENT_VDEV_ENC_SYSFS_PATH/power_status)" == "off" ] ; then + break 2 + fi + sleep 0.1 + done +done + +if [ "$(cat $ZEVENT_VDEV_ENC_SYSFS_PATH/power_status)" != "off" ] ; then + exit 5 +fi + +zed_log_msg "powered down slot $ZEVENT_VDEV_ENC_SYSFS_PATH for $ZEVENT_VDEV_PATH" diff --git a/straper/db/public/zfs/etc-zfs/zed.d/history_event-zfs-list-cacher.sh b/straper/db/public/zfs/etc-zfs/zed.d/history_event-zfs-list-cacher.sh new file mode 100755 index 0000000..cacf921 --- /dev/null +++ b/straper/db/public/zfs/etc-zfs/zed.d/history_event-zfs-list-cacher.sh @@ -0,0 +1,85 @@ +#!/bin/sh +# shellcheck disable=SC2154 +# +# Track changes to enumerated pools for use in early-boot +set -ef + +FSLIST="/etc/zfs/zfs-list.cache/${ZEVENT_POOL}" +FSLIST_TMP="/run/zfs-list.cache@${ZEVENT_POOL}" + +# If the pool specific cache file is not writeable, abort +[ -w "${FSLIST}" ] || exit 0 + +[ -f "${ZED_ZEDLET_DIR}/zed.rc" ] && . "${ZED_ZEDLET_DIR}/zed.rc" +. "${ZED_ZEDLET_DIR}/zed-functions.sh" + +[ "$ZEVENT_SUBCLASS" != "history_event" ] && exit 0 +zed_check_cmd "${ZFS}" sort diff + +# If we are acting on a snapshot, we have nothing to do +[ "${ZEVENT_HISTORY_DSNAME%@*}" = "${ZEVENT_HISTORY_DSNAME}" ] || exit 0 + +# We lock the output file to avoid simultaneous writes. +# If we run into trouble, log and drop the lock +abort_alter() { + zed_log_msg "Error updating zfs-list.cache for ${ZEVENT_POOL}!" + zed_unlock "${FSLIST}" +} + +finished() { + zed_unlock "${FSLIST}" + trap - EXIT + exit 0 +} + +case "${ZEVENT_HISTORY_INTERNAL_NAME}" in + create|"finish receiving"|import|destroy|rename) + ;; + + export) + zed_lock "${FSLIST}" + trap abort_alter EXIT + echo > "${FSLIST}" + finished + ;; + + set|inherit) + # Only act if one of the tracked properties is altered. + case "${ZEVENT_HISTORY_INTERNAL_STR%%=*}" in + canmount|mountpoint|atime|relatime|devices|exec|readonly| \ + setuid|nbmand|encroot|keylocation|org.openzfs.systemd:requires| \ + org.openzfs.systemd:requires-mounts-for| \ + org.openzfs.systemd:before|org.openzfs.systemd:after| \ + org.openzfs.systemd:wanted-by|org.openzfs.systemd:required-by| \ + org.openzfs.systemd:nofail|org.openzfs.systemd:ignore \ + ) ;; + *) exit 0 ;; + esac + ;; + + *) + # Ignore all other events. + exit 0 + ;; +esac + +zed_lock "${FSLIST}" +trap abort_alter EXIT + +PROPS="name,mountpoint,canmount,atime,relatime,devices,exec\ +,readonly,setuid,nbmand,encroot,keylocation\ +,org.openzfs.systemd:requires,org.openzfs.systemd:requires-mounts-for\ +,org.openzfs.systemd:before,org.openzfs.systemd:after\ +,org.openzfs.systemd:wanted-by,org.openzfs.systemd:required-by\ +,org.openzfs.systemd:nofail,org.openzfs.systemd:ignore" + +"${ZFS}" list -H -t filesystem -o "${PROPS}" -r "${ZEVENT_POOL}" > "${FSLIST_TMP}" + +# Sort the output so that it is stable +sort "${FSLIST_TMP}" -o "${FSLIST_TMP}" + +# Don't modify the file if it hasn't changed +diff -q "${FSLIST_TMP}" "${FSLIST}" || cat "${FSLIST_TMP}" > "${FSLIST}" +rm -f "${FSLIST_TMP}" + +finished diff --git a/straper/db/public/zfs/etc-zfs/zed.d/pool_import-led.sh b/straper/db/public/zfs/etc-zfs/zed.d/pool_import-led.sh new file mode 100755 index 0000000..40cb61f --- /dev/null +++ b/straper/db/public/zfs/etc-zfs/zed.d/pool_import-led.sh @@ -0,0 +1,242 @@ +#!/bin/sh +# shellcheck disable=SC2154 +# +# Turn off/on vdevs' enclosure fault LEDs when their pool's state changes. +# +# Turn a vdev's fault LED on if it becomes FAULTED, DEGRADED or UNAVAIL. +# Turn its LED off when it's back ONLINE again. +# +# This script run in two basic modes: +# +# 1. If $ZEVENT_VDEV_ENC_SYSFS_PATH and $ZEVENT_VDEV_STATE_STR are set, then +# only set the LED for that particular vdev. This is the case for statechange +# events and some vdev_* events. +# +# 2. If those vars are not set, then check the state of all vdevs in the pool +# and set the LEDs accordingly. This is the case for pool_import events. +# +# Note that this script requires that your enclosure be supported by the +# Linux SCSI Enclosure services (SES) driver. The script will do nothing +# if you have no enclosure, or if your enclosure isn't supported. +# +# Exit codes: +# 0: enclosure led successfully set +# 1: enclosure leds not available +# 2: enclosure leds administratively disabled +# 3: The led sysfs path passed from ZFS does not exist +# 4: $ZPOOL not set +# 5: awk is not installed + +[ -f "${ZED_ZEDLET_DIR}/zed.rc" ] && . "${ZED_ZEDLET_DIR}/zed.rc" +. "${ZED_ZEDLET_DIR}/zed-functions.sh" + +if [ ! -d /sys/class/enclosure ] && [ ! -d /sys/bus/pci/slots ] ; then + # No JBOD enclosure or NVMe slots + exit 1 +fi + +if [ "${ZED_USE_ENCLOSURE_LEDS}" != "1" ] ; then + exit 2 +fi + +zed_check_cmd "$ZPOOL" || exit 4 +zed_check_cmd awk || exit 5 + +# Global used in set_led debug print +vdev="" + +# check_and_set_led (file, val) +# +# Read an enclosure sysfs file, and write it if it's not already set to 'val' +# +# Arguments +# file: sysfs file to set (like /sys/class/enclosure/0:0:1:0/SLOT 10/fault) +# val: value to set it to +# +# Return +# 0 on success, 3 on missing sysfs path +# +check_and_set_led() +{ + file="$1" + val="$2" + + if [ -z "$val" ]; then + return 0 + fi + + if [ ! -e "$file" ] ; then + return 3 + fi + + # If another process is accessing the LED when we attempt to update it, + # the update will be lost so retry until the LED actually changes or we + # timeout. + for _ in 1 2 3 4 5; do + # We want to check the current state first, since writing to the + # 'fault' entry always causes a SES command, even if the + # current state is already what you want. + read -r current < "${file}" + + # On some enclosures if you write 1 to fault, and read it back, + # it will return 2. Treat all non-zero values as 1 for + # simplicity. + if [ "$current" != "0" ] ; then + current=1 + fi + + if [ "$current" != "$val" ] ; then + echo "$val" > "$file" + zed_log_msg "vdev $vdev set '$file' LED to $val" + else + break + fi + done +} + +# Fault LEDs for JBODs and NVMe drives are handled a little differently. +# +# On JBODs the fault LED is called 'fault' and on a path like this: +# +# /sys/class/enclosure/0:0:1:0/SLOT 10/fault +# +# On NVMe it's called 'attention' and on a path like this: +# +# /sys/bus/pci/slot/0/attention +# +# This function returns the full path to the fault LED file for a given +# enclosure/slot directory. +# +path_to_led() +{ + dir=$1 + if [ -f "$dir/fault" ] ; then + echo "$dir/fault" + elif [ -f "$dir/attention" ] ; then + echo "$dir/attention" + fi +} + +state_to_val() +{ + state="$1" + case "$state" in + FAULTED|DEGRADED|UNAVAIL|REMOVED) + echo 1 + ;; + ONLINE) + echo 0 + ;; + *) + echo "invalid state: $state" + ;; + esac +} + +# +# Given a nvme name like 'nvme0n1', pass back its slot directory +# like "/sys/bus/pci/slots/0" +# +nvme_dev_to_slot() +{ + dev="$1" + + # Get the address "0000:01:00.0" + read -r address < "/sys/class/block/$dev/device/address" + + find /sys/bus/pci/slots -regex '.*/[0-9]+/address$' | \ + while read -r sys_addr; do + read -r this_address < "$sys_addr" + + # The format of address is a little different between + # /sys/class/block/$dev/device/address and + # /sys/bus/pci/slots/ + # + # address= "0000:01:00.0" + # this_address = "0000:01:00" + # + if echo "$address" | grep -Eq ^"$this_address" ; then + echo "${sys_addr%/*}" + break + fi + done +} + + +# process_pool (pool) +# +# Iterate through a pool and set the vdevs' enclosure slot LEDs to +# those vdevs' state. +# +# Arguments +# pool: Pool name. +# +# Return +# 0 on success, 3 on missing sysfs path +# +process_pool() +{ + pool="$1" + + # The output will be the vdevs only (from "grep '/dev/'"): + # + # U45 ONLINE 0 0 0 /dev/sdk 0 + # U46 ONLINE 0 0 0 /dev/sdm 0 + # U47 ONLINE 0 0 0 /dev/sdn 0 + # U50 ONLINE 0 0 0 /dev/sdbn 0 + # + ZPOOL_SCRIPTS_AS_ROOT=1 $ZPOOL status -c upath,fault_led "$pool" | grep '/dev/' | ( + rc=0 + while read -r vdev state _ _ _ therest; do + # Read out current LED value and path + # Get dev name (like 'sda') + dev=$(basename "$(echo "$therest" | awk '{print $(NF-1)}')") + vdev_enc_sysfs_path=$(realpath "/sys/class/block/$dev/device/enclosure_device"*) + if [ ! -d "$vdev_enc_sysfs_path" ] ; then + # This is not a JBOD disk, but it could be a PCI NVMe drive + vdev_enc_sysfs_path=$(nvme_dev_to_slot "$dev") + fi + + current_val=$(echo "$therest" | awk '{print $NF}') + + if [ "$current_val" != "0" ] ; then + current_val=1 + fi + + if [ -z "$vdev_enc_sysfs_path" ] ; then + # Skip anything with no sysfs LED entries + continue + fi + + led_path=$(path_to_led "$vdev_enc_sysfs_path") + if [ ! -e "$led_path" ] ; then + rc=3 + zed_log_msg "vdev $vdev '$led_path' doesn't exist" + continue + fi + + val=$(state_to_val "$state") + + if [ "$current_val" = "$val" ] ; then + # LED is already set correctly + continue + fi + + if ! check_and_set_led "$led_path" "$val"; then + rc=3 + fi + done + exit "$rc"; ) +} + +if [ -n "$ZEVENT_VDEV_ENC_SYSFS_PATH" ] && [ -n "$ZEVENT_VDEV_STATE_STR" ] ; then + # Got a statechange for an individual vdev + val=$(state_to_val "$ZEVENT_VDEV_STATE_STR") + vdev=$(basename "$ZEVENT_VDEV_PATH") + ledpath=$(path_to_led "$ZEVENT_VDEV_ENC_SYSFS_PATH") + check_and_set_led "$ledpath" "$val" +else + # Process the entire pool + poolname=$(zed_guid_to_pool "$ZEVENT_POOL_GUID") + process_pool "$poolname" +fi diff --git a/straper/db/public/zfs/etc-zfs/zed.d/resilver_finish-notify.sh b/straper/db/public/zfs/etc-zfs/zed.d/resilver_finish-notify.sh new file mode 100755 index 0000000..fc9dc23 --- /dev/null +++ b/straper/db/public/zfs/etc-zfs/zed.d/resilver_finish-notify.sh @@ -0,0 +1,60 @@ +#!/bin/sh +# shellcheck disable=SC2154 +# +# Send notification in response to a RESILVER_FINISH or SCRUB_FINISH. +# +# By default, "zpool status" output will only be included for a scrub_finish +# zevent if the pool is not healthy; to always include its output, set +# ZED_NOTIFY_VERBOSE=1. +# +# Exit codes: +# 0: notification sent +# 1: notification failed +# 2: notification not configured +# 3: notification suppressed +# 9: internal error + +[ -f "${ZED_ZEDLET_DIR}/zed.rc" ] && . "${ZED_ZEDLET_DIR}/zed.rc" +. "${ZED_ZEDLET_DIR}/zed-functions.sh" + +[ -n "${ZEVENT_POOL}" ] || exit 9 +[ -n "${ZEVENT_SUBCLASS}" ] || exit 9 + +if [ "${ZEVENT_SUBCLASS}" = "resilver_finish" ]; then + action="resilver" +elif [ "${ZEVENT_SUBCLASS}" = "scrub_finish" ]; then + action="scrub" +else + zed_log_err "unsupported event class \"${ZEVENT_SUBCLASS}\"" + exit 9 +fi + +zed_check_cmd "${ZPOOL}" || exit 9 + +# For scrub, suppress notification if the pool is healthy +# and verbosity is not enabled. +# +if [ "${ZEVENT_SUBCLASS}" = "scrub_finish" ]; then + healthy="$("${ZPOOL}" status -x "${ZEVENT_POOL}" \ + | grep "'${ZEVENT_POOL}' is healthy")" + [ -n "${healthy}" ] && [ "${ZED_NOTIFY_VERBOSE}" -eq 0 ] && exit 3 +fi + +umask 077 +note_subject="ZFS ${ZEVENT_SUBCLASS} event for ${ZEVENT_POOL} on $(hostname)" +note_pathname="$(mktemp)" +{ + echo "ZFS has finished a ${action}:" + echo + echo " eid: ${ZEVENT_EID}" + echo " class: ${ZEVENT_SUBCLASS}" + echo " host: $(hostname)" + echo " time: ${ZEVENT_TIME_STRING}" + + "${ZPOOL}" status "${ZEVENT_POOL}" + +} > "${note_pathname}" + +zed_notify "${note_subject}" "${note_pathname}"; rv=$? +rm -f "${note_pathname}" +exit "${rv}" diff --git a/straper/db/public/zfs/etc-zfs/zed.d/resilver_finish-start-scrub.sh b/straper/db/public/zfs/etc-zfs/zed.d/resilver_finish-start-scrub.sh new file mode 100755 index 0000000..cafce6f --- /dev/null +++ b/straper/db/public/zfs/etc-zfs/zed.d/resilver_finish-start-scrub.sh @@ -0,0 +1,20 @@ +#!/bin/sh +# shellcheck disable=SC2154 +# resilver_finish-start-scrub.sh +# Run a scrub after a resilver +# +# Exit codes: +# 1: Internal error +# 2: Script wasn't enabled in zed.rc +# 3: Scrubs are automatically started for sequential resilvers +[ -f "${ZED_ZEDLET_DIR}/zed.rc" ] && . "${ZED_ZEDLET_DIR}/zed.rc" +. "${ZED_ZEDLET_DIR}/zed-functions.sh" + +[ "${ZED_SCRUB_AFTER_RESILVER}" = "1" ] || exit 2 +[ "${ZEVENT_RESILVER_TYPE}" != "sequential" ] || exit 3 +[ -n "${ZEVENT_POOL}" ] || exit 1 +[ -n "${ZEVENT_SUBCLASS}" ] || exit 1 +zed_check_cmd "${ZPOOL}" || exit 1 + +zed_log_msg "Starting scrub after resilver on ${ZEVENT_POOL}" +"${ZPOOL}" scrub "${ZEVENT_POOL}" diff --git a/straper/db/public/zfs/etc-zfs/zed.d/scrub_finish-notify.sh b/straper/db/public/zfs/etc-zfs/zed.d/scrub_finish-notify.sh new file mode 100755 index 0000000..fc9dc23 --- /dev/null +++ b/straper/db/public/zfs/etc-zfs/zed.d/scrub_finish-notify.sh @@ -0,0 +1,60 @@ +#!/bin/sh +# shellcheck disable=SC2154 +# +# Send notification in response to a RESILVER_FINISH or SCRUB_FINISH. +# +# By default, "zpool status" output will only be included for a scrub_finish +# zevent if the pool is not healthy; to always include its output, set +# ZED_NOTIFY_VERBOSE=1. +# +# Exit codes: +# 0: notification sent +# 1: notification failed +# 2: notification not configured +# 3: notification suppressed +# 9: internal error + +[ -f "${ZED_ZEDLET_DIR}/zed.rc" ] && . "${ZED_ZEDLET_DIR}/zed.rc" +. "${ZED_ZEDLET_DIR}/zed-functions.sh" + +[ -n "${ZEVENT_POOL}" ] || exit 9 +[ -n "${ZEVENT_SUBCLASS}" ] || exit 9 + +if [ "${ZEVENT_SUBCLASS}" = "resilver_finish" ]; then + action="resilver" +elif [ "${ZEVENT_SUBCLASS}" = "scrub_finish" ]; then + action="scrub" +else + zed_log_err "unsupported event class \"${ZEVENT_SUBCLASS}\"" + exit 9 +fi + +zed_check_cmd "${ZPOOL}" || exit 9 + +# For scrub, suppress notification if the pool is healthy +# and verbosity is not enabled. +# +if [ "${ZEVENT_SUBCLASS}" = "scrub_finish" ]; then + healthy="$("${ZPOOL}" status -x "${ZEVENT_POOL}" \ + | grep "'${ZEVENT_POOL}' is healthy")" + [ -n "${healthy}" ] && [ "${ZED_NOTIFY_VERBOSE}" -eq 0 ] && exit 3 +fi + +umask 077 +note_subject="ZFS ${ZEVENT_SUBCLASS} event for ${ZEVENT_POOL} on $(hostname)" +note_pathname="$(mktemp)" +{ + echo "ZFS has finished a ${action}:" + echo + echo " eid: ${ZEVENT_EID}" + echo " class: ${ZEVENT_SUBCLASS}" + echo " host: $(hostname)" + echo " time: ${ZEVENT_TIME_STRING}" + + "${ZPOOL}" status "${ZEVENT_POOL}" + +} > "${note_pathname}" + +zed_notify "${note_subject}" "${note_pathname}"; rv=$? +rm -f "${note_pathname}" +exit "${rv}" diff --git a/straper/db/public/zfs/etc-zfs/zed.d/statechange-led.sh b/straper/db/public/zfs/etc-zfs/zed.d/statechange-led.sh new file mode 100755 index 0000000..40cb61f --- /dev/null +++ b/straper/db/public/zfs/etc-zfs/zed.d/statechange-led.sh @@ -0,0 +1,242 @@ +#!/bin/sh +# shellcheck disable=SC2154 +# +# Turn off/on vdevs' enclosure fault LEDs when their pool's state changes. +# +# Turn a vdev's fault LED on if it becomes FAULTED, DEGRADED or UNAVAIL. +# Turn its LED off when it's back ONLINE again. +# +# This script run in two basic modes: +# +# 1. If $ZEVENT_VDEV_ENC_SYSFS_PATH and $ZEVENT_VDEV_STATE_STR are set, then +# only set the LED for that particular vdev. This is the case for statechange +# events and some vdev_* events. +# +# 2. If those vars are not set, then check the state of all vdevs in the pool +# and set the LEDs accordingly. This is the case for pool_import events. +# +# Note that this script requires that your enclosure be supported by the +# Linux SCSI Enclosure services (SES) driver. The script will do nothing +# if you have no enclosure, or if your enclosure isn't supported. +# +# Exit codes: +# 0: enclosure led successfully set +# 1: enclosure leds not available +# 2: enclosure leds administratively disabled +# 3: The led sysfs path passed from ZFS does not exist +# 4: $ZPOOL not set +# 5: awk is not installed + +[ -f "${ZED_ZEDLET_DIR}/zed.rc" ] && . "${ZED_ZEDLET_DIR}/zed.rc" +. "${ZED_ZEDLET_DIR}/zed-functions.sh" + +if [ ! -d /sys/class/enclosure ] && [ ! -d /sys/bus/pci/slots ] ; then + # No JBOD enclosure or NVMe slots + exit 1 +fi + +if [ "${ZED_USE_ENCLOSURE_LEDS}" != "1" ] ; then + exit 2 +fi + +zed_check_cmd "$ZPOOL" || exit 4 +zed_check_cmd awk || exit 5 + +# Global used in set_led debug print +vdev="" + +# check_and_set_led (file, val) +# +# Read an enclosure sysfs file, and write it if it's not already set to 'val' +# +# Arguments +# file: sysfs file to set (like /sys/class/enclosure/0:0:1:0/SLOT 10/fault) +# val: value to set it to +# +# Return +# 0 on success, 3 on missing sysfs path +# +check_and_set_led() +{ + file="$1" + val="$2" + + if [ -z "$val" ]; then + return 0 + fi + + if [ ! -e "$file" ] ; then + return 3 + fi + + # If another process is accessing the LED when we attempt to update it, + # the update will be lost so retry until the LED actually changes or we + # timeout. + for _ in 1 2 3 4 5; do + # We want to check the current state first, since writing to the + # 'fault' entry always causes a SES command, even if the + # current state is already what you want. + read -r current < "${file}" + + # On some enclosures if you write 1 to fault, and read it back, + # it will return 2. Treat all non-zero values as 1 for + # simplicity. + if [ "$current" != "0" ] ; then + current=1 + fi + + if [ "$current" != "$val" ] ; then + echo "$val" > "$file" + zed_log_msg "vdev $vdev set '$file' LED to $val" + else + break + fi + done +} + +# Fault LEDs for JBODs and NVMe drives are handled a little differently. +# +# On JBODs the fault LED is called 'fault' and on a path like this: +# +# /sys/class/enclosure/0:0:1:0/SLOT 10/fault +# +# On NVMe it's called 'attention' and on a path like this: +# +# /sys/bus/pci/slot/0/attention +# +# This function returns the full path to the fault LED file for a given +# enclosure/slot directory. +# +path_to_led() +{ + dir=$1 + if [ -f "$dir/fault" ] ; then + echo "$dir/fault" + elif [ -f "$dir/attention" ] ; then + echo "$dir/attention" + fi +} + +state_to_val() +{ + state="$1" + case "$state" in + FAULTED|DEGRADED|UNAVAIL|REMOVED) + echo 1 + ;; + ONLINE) + echo 0 + ;; + *) + echo "invalid state: $state" + ;; + esac +} + +# +# Given a nvme name like 'nvme0n1', pass back its slot directory +# like "/sys/bus/pci/slots/0" +# +nvme_dev_to_slot() +{ + dev="$1" + + # Get the address "0000:01:00.0" + read -r address < "/sys/class/block/$dev/device/address" + + find /sys/bus/pci/slots -regex '.*/[0-9]+/address$' | \ + while read -r sys_addr; do + read -r this_address < "$sys_addr" + + # The format of address is a little different between + # /sys/class/block/$dev/device/address and + # /sys/bus/pci/slots/ + # + # address= "0000:01:00.0" + # this_address = "0000:01:00" + # + if echo "$address" | grep -Eq ^"$this_address" ; then + echo "${sys_addr%/*}" + break + fi + done +} + + +# process_pool (pool) +# +# Iterate through a pool and set the vdevs' enclosure slot LEDs to +# those vdevs' state. +# +# Arguments +# pool: Pool name. +# +# Return +# 0 on success, 3 on missing sysfs path +# +process_pool() +{ + pool="$1" + + # The output will be the vdevs only (from "grep '/dev/'"): + # + # U45 ONLINE 0 0 0 /dev/sdk 0 + # U46 ONLINE 0 0 0 /dev/sdm 0 + # U47 ONLINE 0 0 0 /dev/sdn 0 + # U50 ONLINE 0 0 0 /dev/sdbn 0 + # + ZPOOL_SCRIPTS_AS_ROOT=1 $ZPOOL status -c upath,fault_led "$pool" | grep '/dev/' | ( + rc=0 + while read -r vdev state _ _ _ therest; do + # Read out current LED value and path + # Get dev name (like 'sda') + dev=$(basename "$(echo "$therest" | awk '{print $(NF-1)}')") + vdev_enc_sysfs_path=$(realpath "/sys/class/block/$dev/device/enclosure_device"*) + if [ ! -d "$vdev_enc_sysfs_path" ] ; then + # This is not a JBOD disk, but it could be a PCI NVMe drive + vdev_enc_sysfs_path=$(nvme_dev_to_slot "$dev") + fi + + current_val=$(echo "$therest" | awk '{print $NF}') + + if [ "$current_val" != "0" ] ; then + current_val=1 + fi + + if [ -z "$vdev_enc_sysfs_path" ] ; then + # Skip anything with no sysfs LED entries + continue + fi + + led_path=$(path_to_led "$vdev_enc_sysfs_path") + if [ ! -e "$led_path" ] ; then + rc=3 + zed_log_msg "vdev $vdev '$led_path' doesn't exist" + continue + fi + + val=$(state_to_val "$state") + + if [ "$current_val" = "$val" ] ; then + # LED is already set correctly + continue + fi + + if ! check_and_set_led "$led_path" "$val"; then + rc=3 + fi + done + exit "$rc"; ) +} + +if [ -n "$ZEVENT_VDEV_ENC_SYSFS_PATH" ] && [ -n "$ZEVENT_VDEV_STATE_STR" ] ; then + # Got a statechange for an individual vdev + val=$(state_to_val "$ZEVENT_VDEV_STATE_STR") + vdev=$(basename "$ZEVENT_VDEV_PATH") + ledpath=$(path_to_led "$ZEVENT_VDEV_ENC_SYSFS_PATH") + check_and_set_led "$ledpath" "$val" +else + # Process the entire pool + poolname=$(zed_guid_to_pool "$ZEVENT_POOL_GUID") + process_pool "$poolname" +fi diff --git a/straper/db/public/zfs/etc-zfs/zed.d/statechange-notify.sh b/straper/db/public/zfs/etc-zfs/zed.d/statechange-notify.sh new file mode 100755 index 0000000..c8c8323 --- /dev/null +++ b/straper/db/public/zfs/etc-zfs/zed.d/statechange-notify.sh @@ -0,0 +1,77 @@ +#!/bin/sh +# SPDX-License-Identifier: CDDL-1.0 +# shellcheck disable=SC2154 +# +# CDDL HEADER START +# +# The contents of this file are subject to the terms of the +# Common Development and Distribution License Version 1.0 (CDDL-1.0). +# You can obtain a copy of the license from the top-level file +# "OPENSOLARIS.LICENSE" or at <http://opensource.org/licenses/CDDL-1.0>. +# You may not use this file except in compliance with the license. +# +# CDDL HEADER END +# + +# +# Send notification in response to a fault induced statechange +# +# ZEVENT_SUBCLASS: 'statechange' +# ZEVENT_VDEV_STATE_STR: 'DEGRADED', 'FAULTED', 'REMOVED', or 'UNAVAIL' +# +# Exit codes: +# 0: notification sent +# 1: notification failed +# 2: notification not configured +# 3: statechange not relevant +# 4: statechange string missing (unexpected) + +[ -f "${ZED_ZEDLET_DIR}/zed.rc" ] && . "${ZED_ZEDLET_DIR}/zed.rc" +. "${ZED_ZEDLET_DIR}/zed-functions.sh" + +[ -n "${ZEVENT_VDEV_STATE_STR}" ] || exit 4 + +if [ "${ZEVENT_VDEV_STATE_STR}" != "FAULTED" ] \ + && [ "${ZEVENT_VDEV_STATE_STR}" != "DEGRADED" ] \ + && [ "${ZEVENT_VDEV_STATE_STR}" != "REMOVED" ] \ + && [ "${ZEVENT_VDEV_STATE_STR}" != "UNAVAIL" ]; then + exit 3 +fi + +umask 077 +note_subject="ZFS device fault for pool ${ZEVENT_POOL} on $(hostname)" +note_pathname="$(mktemp)" +{ + if [ "${ZEVENT_VDEV_STATE_STR}" = "FAULTED" ] ; then + echo "The number of I/O errors associated with a ZFS device exceeded" + echo "acceptable levels. ZFS has marked the device as faulted." + elif [ "${ZEVENT_VDEV_STATE_STR}" = "DEGRADED" ] ; then + echo "The number of checksum errors associated with a ZFS device" + echo "exceeded acceptable levels. ZFS has marked the device as" + echo "degraded." + else + echo "ZFS has detected that a device was removed." + fi + + echo + echo " impact: Fault tolerance of the pool may be compromised." + echo " eid: ${ZEVENT_EID}" + echo " class: ${ZEVENT_SUBCLASS}" + echo " state: ${ZEVENT_VDEV_STATE_STR}" + echo " host: $(hostname)" + echo " time: ${ZEVENT_TIME_STRING}" + + [ -n "${ZEVENT_VDEV_TYPE}" ] && echo " vtype: ${ZEVENT_VDEV_TYPE}" + [ -n "${ZEVENT_VDEV_PATH}" ] && echo " vpath: ${ZEVENT_VDEV_PATH}" + [ -n "${ZEVENT_VDEV_PHYSPATH}" ] && echo " vphys: ${ZEVENT_VDEV_PHYSPATH}" + [ -n "${ZEVENT_VDEV_GUID}" ] && echo " vguid: ${ZEVENT_VDEV_GUID}" + [ -n "${ZEVENT_VDEV_DEVID}" ] && echo " devid: ${ZEVENT_VDEV_DEVID}" + + echo " pool: ${ZEVENT_POOL} (${ZEVENT_POOL_GUID})" + +} > "${note_pathname}" + +zed_notify "${note_subject}" "${note_pathname}"; rv=$? + +rm -f "${note_pathname}" +exit "${rv}" diff --git a/straper/db/public/zfs/etc-zfs/zed.d/statechange-slot_off.sh b/straper/db/public/zfs/etc-zfs/zed.d/statechange-slot_off.sh new file mode 100755 index 0000000..06acce9 --- /dev/null +++ b/straper/db/public/zfs/etc-zfs/zed.d/statechange-slot_off.sh @@ -0,0 +1,64 @@ +#!/bin/sh +# shellcheck disable=SC3014,SC2154,SC2086,SC2034 +# +# Turn off disk's enclosure slot if it becomes FAULTED. +# +# Bad SCSI disks can often "disappear and reappear" causing all sorts of chaos +# as they flip between FAULTED and ONLINE. If +# ZED_POWER_OFF_ENCLOSURE_SLOT_ON_FAULT is set in zed.rc, and the disk gets +# FAULTED, then power down the slot via sysfs: +# +# /sys/class/enclosure/<enclosure>/<slot>/power_status +# +# We assume the user will be responsible for turning the slot back on again. +# +# Note that this script requires that your enclosure be supported by the +# Linux SCSI Enclosure services (SES) driver. The script will do nothing +# if you have no enclosure, or if your enclosure isn't supported. +# +# Exit codes: +# 0: slot successfully powered off +# 1: enclosure not available +# 2: ZED_POWER_OFF_ENCLOSURE_SLOT_ON_FAULT disabled +# 3: vdev was not FAULTED +# 4: The enclosure sysfs path passed from ZFS does not exist +# 5: Enclosure slot didn't actually turn off after we told it to + +[ -f "${ZED_ZEDLET_DIR}/zed.rc" ] && . "${ZED_ZEDLET_DIR}/zed.rc" +. "${ZED_ZEDLET_DIR}/zed-functions.sh" + +if [ ! -d /sys/class/enclosure ] ; then + # No JBOD enclosure or NVMe slots + exit 1 +fi + +if [ "${ZED_POWER_OFF_ENCLOSURE_SLOT_ON_FAULT}" != "1" ] ; then + exit 2 +fi + +if [ "$ZEVENT_VDEV_STATE_STR" != "FAULTED" ] ; then + exit 3 +fi + +if [ ! -f "$ZEVENT_VDEV_ENC_SYSFS_PATH/power_status" ] ; then + exit 4 +fi + +# Turn off the slot and wait for sysfs to report that the slot is off. +# It can take ~400ms on some enclosures and multiple retries may be needed. +for i in $(seq 1 20) ; do + echo "off" | tee "$ZEVENT_VDEV_ENC_SYSFS_PATH/power_status" + + for j in $(seq 1 5) ; do + if [ "$(cat $ZEVENT_VDEV_ENC_SYSFS_PATH/power_status)" == "off" ] ; then + break 2 + fi + sleep 0.1 + done +done + +if [ "$(cat $ZEVENT_VDEV_ENC_SYSFS_PATH/power_status)" != "off" ] ; then + exit 5 +fi + +zed_log_msg "powered down slot $ZEVENT_VDEV_ENC_SYSFS_PATH for $ZEVENT_VDEV_PATH" diff --git a/straper/db/public/zfs/etc-zfs/zed.d/vdev_attach-led.sh b/straper/db/public/zfs/etc-zfs/zed.d/vdev_attach-led.sh new file mode 100755 index 0000000..40cb61f --- /dev/null +++ b/straper/db/public/zfs/etc-zfs/zed.d/vdev_attach-led.sh @@ -0,0 +1,242 @@ +#!/bin/sh +# shellcheck disable=SC2154 +# +# Turn off/on vdevs' enclosure fault LEDs when their pool's state changes. +# +# Turn a vdev's fault LED on if it becomes FAULTED, DEGRADED or UNAVAIL. +# Turn its LED off when it's back ONLINE again. +# +# This script run in two basic modes: +# +# 1. If $ZEVENT_VDEV_ENC_SYSFS_PATH and $ZEVENT_VDEV_STATE_STR are set, then +# only set the LED for that particular vdev. This is the case for statechange +# events and some vdev_* events. +# +# 2. If those vars are not set, then check the state of all vdevs in the pool +# and set the LEDs accordingly. This is the case for pool_import events. +# +# Note that this script requires that your enclosure be supported by the +# Linux SCSI Enclosure services (SES) driver. The script will do nothing +# if you have no enclosure, or if your enclosure isn't supported. +# +# Exit codes: +# 0: enclosure led successfully set +# 1: enclosure leds not available +# 2: enclosure leds administratively disabled +# 3: The led sysfs path passed from ZFS does not exist +# 4: $ZPOOL not set +# 5: awk is not installed + +[ -f "${ZED_ZEDLET_DIR}/zed.rc" ] && . "${ZED_ZEDLET_DIR}/zed.rc" +. "${ZED_ZEDLET_DIR}/zed-functions.sh" + +if [ ! -d /sys/class/enclosure ] && [ ! -d /sys/bus/pci/slots ] ; then + # No JBOD enclosure or NVMe slots + exit 1 +fi + +if [ "${ZED_USE_ENCLOSURE_LEDS}" != "1" ] ; then + exit 2 +fi + +zed_check_cmd "$ZPOOL" || exit 4 +zed_check_cmd awk || exit 5 + +# Global used in set_led debug print +vdev="" + +# check_and_set_led (file, val) +# +# Read an enclosure sysfs file, and write it if it's not already set to 'val' +# +# Arguments +# file: sysfs file to set (like /sys/class/enclosure/0:0:1:0/SLOT 10/fault) +# val: value to set it to +# +# Return +# 0 on success, 3 on missing sysfs path +# +check_and_set_led() +{ + file="$1" + val="$2" + + if [ -z "$val" ]; then + return 0 + fi + + if [ ! -e "$file" ] ; then + return 3 + fi + + # If another process is accessing the LED when we attempt to update it, + # the update will be lost so retry until the LED actually changes or we + # timeout. + for _ in 1 2 3 4 5; do + # We want to check the current state first, since writing to the + # 'fault' entry always causes a SES command, even if the + # current state is already what you want. + read -r current < "${file}" + + # On some enclosures if you write 1 to fault, and read it back, + # it will return 2. Treat all non-zero values as 1 for + # simplicity. + if [ "$current" != "0" ] ; then + current=1 + fi + + if [ "$current" != "$val" ] ; then + echo "$val" > "$file" + zed_log_msg "vdev $vdev set '$file' LED to $val" + else + break + fi + done +} + +# Fault LEDs for JBODs and NVMe drives are handled a little differently. +# +# On JBODs the fault LED is called 'fault' and on a path like this: +# +# /sys/class/enclosure/0:0:1:0/SLOT 10/fault +# +# On NVMe it's called 'attention' and on a path like this: +# +# /sys/bus/pci/slot/0/attention +# +# This function returns the full path to the fault LED file for a given +# enclosure/slot directory. +# +path_to_led() +{ + dir=$1 + if [ -f "$dir/fault" ] ; then + echo "$dir/fault" + elif [ -f "$dir/attention" ] ; then + echo "$dir/attention" + fi +} + +state_to_val() +{ + state="$1" + case "$state" in + FAULTED|DEGRADED|UNAVAIL|REMOVED) + echo 1 + ;; + ONLINE) + echo 0 + ;; + *) + echo "invalid state: $state" + ;; + esac +} + +# +# Given a nvme name like 'nvme0n1', pass back its slot directory +# like "/sys/bus/pci/slots/0" +# +nvme_dev_to_slot() +{ + dev="$1" + + # Get the address "0000:01:00.0" + read -r address < "/sys/class/block/$dev/device/address" + + find /sys/bus/pci/slots -regex '.*/[0-9]+/address$' | \ + while read -r sys_addr; do + read -r this_address < "$sys_addr" + + # The format of address is a little different between + # /sys/class/block/$dev/device/address and + # /sys/bus/pci/slots/ + # + # address= "0000:01:00.0" + # this_address = "0000:01:00" + # + if echo "$address" | grep -Eq ^"$this_address" ; then + echo "${sys_addr%/*}" + break + fi + done +} + + +# process_pool (pool) +# +# Iterate through a pool and set the vdevs' enclosure slot LEDs to +# those vdevs' state. +# +# Arguments +# pool: Pool name. +# +# Return +# 0 on success, 3 on missing sysfs path +# +process_pool() +{ + pool="$1" + + # The output will be the vdevs only (from "grep '/dev/'"): + # + # U45 ONLINE 0 0 0 /dev/sdk 0 + # U46 ONLINE 0 0 0 /dev/sdm 0 + # U47 ONLINE 0 0 0 /dev/sdn 0 + # U50 ONLINE 0 0 0 /dev/sdbn 0 + # + ZPOOL_SCRIPTS_AS_ROOT=1 $ZPOOL status -c upath,fault_led "$pool" | grep '/dev/' | ( + rc=0 + while read -r vdev state _ _ _ therest; do + # Read out current LED value and path + # Get dev name (like 'sda') + dev=$(basename "$(echo "$therest" | awk '{print $(NF-1)}')") + vdev_enc_sysfs_path=$(realpath "/sys/class/block/$dev/device/enclosure_device"*) + if [ ! -d "$vdev_enc_sysfs_path" ] ; then + # This is not a JBOD disk, but it could be a PCI NVMe drive + vdev_enc_sysfs_path=$(nvme_dev_to_slot "$dev") + fi + + current_val=$(echo "$therest" | awk '{print $NF}') + + if [ "$current_val" != "0" ] ; then + current_val=1 + fi + + if [ -z "$vdev_enc_sysfs_path" ] ; then + # Skip anything with no sysfs LED entries + continue + fi + + led_path=$(path_to_led "$vdev_enc_sysfs_path") + if [ ! -e "$led_path" ] ; then + rc=3 + zed_log_msg "vdev $vdev '$led_path' doesn't exist" + continue + fi + + val=$(state_to_val "$state") + + if [ "$current_val" = "$val" ] ; then + # LED is already set correctly + continue + fi + + if ! check_and_set_led "$led_path" "$val"; then + rc=3 + fi + done + exit "$rc"; ) +} + +if [ -n "$ZEVENT_VDEV_ENC_SYSFS_PATH" ] && [ -n "$ZEVENT_VDEV_STATE_STR" ] ; then + # Got a statechange for an individual vdev + val=$(state_to_val "$ZEVENT_VDEV_STATE_STR") + vdev=$(basename "$ZEVENT_VDEV_PATH") + ledpath=$(path_to_led "$ZEVENT_VDEV_ENC_SYSFS_PATH") + check_and_set_led "$ledpath" "$val" +else + # Process the entire pool + poolname=$(zed_guid_to_pool "$ZEVENT_POOL_GUID") + process_pool "$poolname" +fi diff --git a/straper/db/public/zfs/etc-zfs/zed.d/vdev_clear-led.sh b/straper/db/public/zfs/etc-zfs/zed.d/vdev_clear-led.sh new file mode 100755 index 0000000..40cb61f --- /dev/null +++ b/straper/db/public/zfs/etc-zfs/zed.d/vdev_clear-led.sh @@ -0,0 +1,242 @@ +#!/bin/sh +# shellcheck disable=SC2154 +# +# Turn off/on vdevs' enclosure fault LEDs when their pool's state changes. +# +# Turn a vdev's fault LED on if it becomes FAULTED, DEGRADED or UNAVAIL. +# Turn its LED off when it's back ONLINE again. +# +# This script run in two basic modes: +# +# 1. If $ZEVENT_VDEV_ENC_SYSFS_PATH and $ZEVENT_VDEV_STATE_STR are set, then +# only set the LED for that particular vdev. This is the case for statechange +# events and some vdev_* events. +# +# 2. If those vars are not set, then check the state of all vdevs in the pool +# and set the LEDs accordingly. This is the case for pool_import events. +# +# Note that this script requires that your enclosure be supported by the +# Linux SCSI Enclosure services (SES) driver. The script will do nothing +# if you have no enclosure, or if your enclosure isn't supported. +# +# Exit codes: +# 0: enclosure led successfully set +# 1: enclosure leds not available +# 2: enclosure leds administratively disabled +# 3: The led sysfs path passed from ZFS does not exist +# 4: $ZPOOL not set +# 5: awk is not installed + +[ -f "${ZED_ZEDLET_DIR}/zed.rc" ] && . "${ZED_ZEDLET_DIR}/zed.rc" +. "${ZED_ZEDLET_DIR}/zed-functions.sh" + +if [ ! -d /sys/class/enclosure ] && [ ! -d /sys/bus/pci/slots ] ; then + # No JBOD enclosure or NVMe slots + exit 1 +fi + +if [ "${ZED_USE_ENCLOSURE_LEDS}" != "1" ] ; then + exit 2 +fi + +zed_check_cmd "$ZPOOL" || exit 4 +zed_check_cmd awk || exit 5 + +# Global used in set_led debug print +vdev="" + +# check_and_set_led (file, val) +# +# Read an enclosure sysfs file, and write it if it's not already set to 'val' +# +# Arguments +# file: sysfs file to set (like /sys/class/enclosure/0:0:1:0/SLOT 10/fault) +# val: value to set it to +# +# Return +# 0 on success, 3 on missing sysfs path +# +check_and_set_led() +{ + file="$1" + val="$2" + + if [ -z "$val" ]; then + return 0 + fi + + if [ ! -e "$file" ] ; then + return 3 + fi + + # If another process is accessing the LED when we attempt to update it, + # the update will be lost so retry until the LED actually changes or we + # timeout. + for _ in 1 2 3 4 5; do + # We want to check the current state first, since writing to the + # 'fault' entry always causes a SES command, even if the + # current state is already what you want. + read -r current < "${file}" + + # On some enclosures if you write 1 to fault, and read it back, + # it will return 2. Treat all non-zero values as 1 for + # simplicity. + if [ "$current" != "0" ] ; then + current=1 + fi + + if [ "$current" != "$val" ] ; then + echo "$val" > "$file" + zed_log_msg "vdev $vdev set '$file' LED to $val" + else + break + fi + done +} + +# Fault LEDs for JBODs and NVMe drives are handled a little differently. +# +# On JBODs the fault LED is called 'fault' and on a path like this: +# +# /sys/class/enclosure/0:0:1:0/SLOT 10/fault +# +# On NVMe it's called 'attention' and on a path like this: +# +# /sys/bus/pci/slot/0/attention +# +# This function returns the full path to the fault LED file for a given +# enclosure/slot directory. +# +path_to_led() +{ + dir=$1 + if [ -f "$dir/fault" ] ; then + echo "$dir/fault" + elif [ -f "$dir/attention" ] ; then + echo "$dir/attention" + fi +} + +state_to_val() +{ + state="$1" + case "$state" in + FAULTED|DEGRADED|UNAVAIL|REMOVED) + echo 1 + ;; + ONLINE) + echo 0 + ;; + *) + echo "invalid state: $state" + ;; + esac +} + +# +# Given a nvme name like 'nvme0n1', pass back its slot directory +# like "/sys/bus/pci/slots/0" +# +nvme_dev_to_slot() +{ + dev="$1" + + # Get the address "0000:01:00.0" + read -r address < "/sys/class/block/$dev/device/address" + + find /sys/bus/pci/slots -regex '.*/[0-9]+/address$' | \ + while read -r sys_addr; do + read -r this_address < "$sys_addr" + + # The format of address is a little different between + # /sys/class/block/$dev/device/address and + # /sys/bus/pci/slots/ + # + # address= "0000:01:00.0" + # this_address = "0000:01:00" + # + if echo "$address" | grep -Eq ^"$this_address" ; then + echo "${sys_addr%/*}" + break + fi + done +} + + +# process_pool (pool) +# +# Iterate through a pool and set the vdevs' enclosure slot LEDs to +# those vdevs' state. +# +# Arguments +# pool: Pool name. +# +# Return +# 0 on success, 3 on missing sysfs path +# +process_pool() +{ + pool="$1" + + # The output will be the vdevs only (from "grep '/dev/'"): + # + # U45 ONLINE 0 0 0 /dev/sdk 0 + # U46 ONLINE 0 0 0 /dev/sdm 0 + # U47 ONLINE 0 0 0 /dev/sdn 0 + # U50 ONLINE 0 0 0 /dev/sdbn 0 + # + ZPOOL_SCRIPTS_AS_ROOT=1 $ZPOOL status -c upath,fault_led "$pool" | grep '/dev/' | ( + rc=0 + while read -r vdev state _ _ _ therest; do + # Read out current LED value and path + # Get dev name (like 'sda') + dev=$(basename "$(echo "$therest" | awk '{print $(NF-1)}')") + vdev_enc_sysfs_path=$(realpath "/sys/class/block/$dev/device/enclosure_device"*) + if [ ! -d "$vdev_enc_sysfs_path" ] ; then + # This is not a JBOD disk, but it could be a PCI NVMe drive + vdev_enc_sysfs_path=$(nvme_dev_to_slot "$dev") + fi + + current_val=$(echo "$therest" | awk '{print $NF}') + + if [ "$current_val" != "0" ] ; then + current_val=1 + fi + + if [ -z "$vdev_enc_sysfs_path" ] ; then + # Skip anything with no sysfs LED entries + continue + fi + + led_path=$(path_to_led "$vdev_enc_sysfs_path") + if [ ! -e "$led_path" ] ; then + rc=3 + zed_log_msg "vdev $vdev '$led_path' doesn't exist" + continue + fi + + val=$(state_to_val "$state") + + if [ "$current_val" = "$val" ] ; then + # LED is already set correctly + continue + fi + + if ! check_and_set_led "$led_path" "$val"; then + rc=3 + fi + done + exit "$rc"; ) +} + +if [ -n "$ZEVENT_VDEV_ENC_SYSFS_PATH" ] && [ -n "$ZEVENT_VDEV_STATE_STR" ] ; then + # Got a statechange for an individual vdev + val=$(state_to_val "$ZEVENT_VDEV_STATE_STR") + vdev=$(basename "$ZEVENT_VDEV_PATH") + ledpath=$(path_to_led "$ZEVENT_VDEV_ENC_SYSFS_PATH") + check_and_set_led "$ledpath" "$val" +else + # Process the entire pool + poolname=$(zed_guid_to_pool "$ZEVENT_POOL_GUID") + process_pool "$poolname" +fi diff --git a/straper/db/public/zfs/etc-zfs/zed.d/zed-functions.sh b/straper/db/public/zfs/etc-zfs/zed.d/zed-functions.sh new file mode 100644 index 0000000..6e00f15 --- /dev/null +++ b/straper/db/public/zfs/etc-zfs/zed.d/zed-functions.sh @@ -0,0 +1,824 @@ +#!/bin/sh +# shellcheck disable=SC2154,SC3043 +# zed-functions.sh +# +# ZED helper functions for use in ZEDLETs + + +# Variable Defaults +# +: "${ZED_LOCKDIR:="/var/lock"}" +: "${ZED_NOTIFY_INTERVAL_SECS:=3600}" +: "${ZED_NOTIFY_VERBOSE:=0}" +: "${ZED_RUNDIR:="/var/run"}" +: "${ZED_SYSLOG_PRIORITY:="daemon.notice"}" +: "${ZED_SYSLOG_TAG:="zed"}" + +ZED_FLOCK_FD=8 + + +# zed_check_cmd (cmd, ...) +# +# For each argument given, search PATH for the executable command [cmd]. +# Log a message if [cmd] is not found. +# +# Arguments +# cmd: name of executable command for which to search +# +# Return +# 0 if all commands are found in PATH and are executable +# n for a count of the command executables that are not found +# +zed_check_cmd() +{ + local cmd + local rv=0 + + for cmd; do + if ! command -v "${cmd}" >/dev/null 2>&1; then + zed_log_err "\"${cmd}\" not installed" + rv=$((rv + 1)) + fi + done + return "${rv}" +} + + +# zed_log_msg (msg, ...) +# +# Write all argument strings to the system log. +# +# Globals +# ZED_SYSLOG_PRIORITY +# ZED_SYSLOG_TAG +# +# Return +# nothing +# +zed_log_msg() +{ + logger -p "${ZED_SYSLOG_PRIORITY}" -t "${ZED_SYSLOG_TAG}" -- "$@" +} + + +# zed_log_err (msg, ...) +# +# Write an error message to the system log. This message will contain the +# script name, EID, and all argument strings. +# +# Globals +# ZED_SYSLOG_PRIORITY +# ZED_SYSLOG_TAG +# ZEVENT_EID +# +# Return +# nothing +# +zed_log_err() +{ + zed_log_msg "error: ${0##*/}:""${ZEVENT_EID:+" eid=${ZEVENT_EID}:"}" "$@" +} + + +# zed_lock (lockfile, [fd]) +# +# Obtain an exclusive (write) lock on [lockfile]. If the lock cannot be +# immediately acquired, wait until it becomes available. +# +# Every zed_lock() must be paired with a corresponding zed_unlock(). +# +# By default, flock-style locks associate the lockfile with file descriptor 8. +# The bash manpage warns that file descriptors >9 should be used with care as +# they may conflict with file descriptors used internally by the shell. File +# descriptor 9 is reserved for zed_rate_limit(). If concurrent locks are held +# within the same process, they must use different file descriptors (preferably +# decrementing from 8); otherwise, obtaining a new lock with a given file +# descriptor will release the previous lock associated with that descriptor. +# +# Arguments +# lockfile: pathname of the lock file; the lock will be stored in +# ZED_LOCKDIR unless the pathname contains a "/". +# fd: integer for the file descriptor used by flock (OPTIONAL unless holding +# concurrent locks) +# +# Globals +# ZED_FLOCK_FD +# ZED_LOCKDIR +# +# Return +# nothing +# +zed_lock() +{ + local lockfile="$1" + local fd="${2:-${ZED_FLOCK_FD}}" + local umask_bak + local err + + [ -n "${lockfile}" ] || return + if ! expr "${lockfile}" : '.*/' >/dev/null 2>&1; then + lockfile="${ZED_LOCKDIR}/${lockfile}" + fi + + umask_bak="$(umask)" + umask 077 + + # Obtain a lock on the file bound to the given file descriptor. + # + eval "exec ${fd}>> '${lockfile}'" + if ! err="$(flock --exclusive "${fd}" 2>&1)"; then + zed_log_err "failed to lock \"${lockfile}\": ${err}" + fi + + umask "${umask_bak}" +} + + +# zed_unlock (lockfile, [fd]) +# +# Release the lock on [lockfile]. +# +# Arguments +# lockfile: pathname of the lock file +# fd: integer for the file descriptor used by flock (must match the file +# descriptor passed to the zed_lock function call) +# +# Globals +# ZED_FLOCK_FD +# ZED_LOCKDIR +# +# Return +# nothing +# +zed_unlock() +{ + local lockfile="$1" + local fd="${2:-${ZED_FLOCK_FD}}" + local err + + [ -n "${lockfile}" ] || return + if ! expr "${lockfile}" : '.*/' >/dev/null 2>&1; then + lockfile="${ZED_LOCKDIR}/${lockfile}" + fi + + # Release the lock and close the file descriptor. + if ! err="$(flock --unlock "${fd}" 2>&1)"; then + zed_log_err "failed to unlock \"${lockfile}\": ${err}" + fi + eval "exec ${fd}>&-" +} + + +# zed_notify (subject, pathname) +# +# Send a notification via all available methods. +# +# Arguments +# subject: notification subject +# pathname: pathname containing the notification message (OPTIONAL) +# +# Return +# 0: notification succeeded via at least one method +# 1: notification failed +# 2: no notification methods configured +# +zed_notify() +{ + local subject="$1" + local pathname="$2" + local num_success=0 + local num_failure=0 + + zed_notify_email "${subject}" "${pathname}"; rv=$? + [ "${rv}" -eq 0 ] && num_success=$((num_success + 1)) + [ "${rv}" -eq 1 ] && num_failure=$((num_failure + 1)) + + zed_notify_pushbullet "${subject}" "${pathname}"; rv=$? + [ "${rv}" -eq 0 ] && num_success=$((num_success + 1)) + [ "${rv}" -eq 1 ] && num_failure=$((num_failure + 1)) + + zed_notify_slack_webhook "${subject}" "${pathname}"; rv=$? + [ "${rv}" -eq 0 ] && num_success=$((num_success + 1)) + [ "${rv}" -eq 1 ] && num_failure=$((num_failure + 1)) + + zed_notify_pushover "${subject}" "${pathname}"; rv=$? + [ "${rv}" -eq 0 ] && num_success=$((num_success + 1)) + [ "${rv}" -eq 1 ] && num_failure=$((num_failure + 1)) + + zed_notify_ntfy "${subject}" "${pathname}"; rv=$? + [ "${rv}" -eq 0 ] && num_success=$((num_success + 1)) + [ "${rv}" -eq 1 ] && num_failure=$((num_failure + 1)) + + zed_notify_gotify "${subject}" "${pathname}"; rv=$? + [ "${rv}" -eq 0 ] && num_success=$((num_success + 1)) + [ "${rv}" -eq 1 ] && num_failure=$((num_failure + 1)) + + [ "${num_success}" -gt 0 ] && return 0 + [ "${num_failure}" -gt 0 ] && return 1 + return 2 +} + + +# zed_notify_email (subject, pathname) +# +# Send a notification via email to the address specified by ZED_EMAIL_ADDR. +# +# Requires the mail executable to be installed in the standard PATH, or +# ZED_EMAIL_PROG to be defined with the pathname of an executable capable of +# reading a message body from stdin. +# +# Command-line options to the mail executable can be specified in +# ZED_EMAIL_OPTS. This undergoes the following keyword substitutions: +# - @ADDRESS@ is replaced with the space-delimited recipient email address(es) +# - @SUBJECT@ is replaced with the notification subject +# If @SUBJECT@ was omited here, a "Subject: ..." header will be added to notification +# +# +# Arguments +# subject: notification subject +# pathname: pathname containing the notification message (OPTIONAL) +# +# Globals +# ZED_EMAIL_PROG +# ZED_EMAIL_OPTS +# ZED_EMAIL_ADDR +# +# Return +# 0: notification sent +# 1: notification failed +# 2: not configured +# +zed_notify_email() +{ + local subject="${1:-"ZED notification"}" + local pathname="${2:-"/dev/null"}" + + : "${ZED_EMAIL_PROG:="mail"}" + : "${ZED_EMAIL_OPTS:="-s '@SUBJECT@' @ADDRESS@"}" + + # For backward compatibility with ZED_EMAIL. + if [ -n "${ZED_EMAIL}" ] && [ -z "${ZED_EMAIL_ADDR}" ]; then + ZED_EMAIL_ADDR="${ZED_EMAIL}" + fi + [ -n "${ZED_EMAIL_ADDR}" ] || return 2 + + zed_check_cmd "${ZED_EMAIL_PROG}" || return 1 + + [ -n "${subject}" ] || return 1 + if [ ! -r "${pathname}" ]; then + zed_log_err \ + "${ZED_EMAIL_PROG##*/} cannot read \"${pathname}\"" + return 1 + fi + + # construct cmdline options + ZED_EMAIL_OPTS_PARSED="$(echo "${ZED_EMAIL_OPTS}" \ + | sed -e "s/@ADDRESS@/${ZED_EMAIL_ADDR}/g" \ + -e "s/@SUBJECT@/${subject}/g")" + + # pipe message to email prog + # shellcheck disable=SC2086,SC2248 + { + # no subject passed as option? + if [ "${ZED_EMAIL_OPTS%@SUBJECT@*}" = "${ZED_EMAIL_OPTS}" ] ; then + # inject subject header + printf "Subject: %s\n" "${subject}" + # The following empty line is needed to separate the header from the + # body of the message. Otherwise programs like sendmail will skip + # everything up to the first empty line (or wont send an email at + # all) and will still exit with exit code 0 + printf "\n" + fi + # output message + cat "${pathname}" + } | + eval ${ZED_EMAIL_PROG} ${ZED_EMAIL_OPTS_PARSED} >/dev/null 2>&1 + rv=$? + if [ "${rv}" -ne 0 ]; then + zed_log_err "${ZED_EMAIL_PROG##*/} exit=${rv}" + return 1 + fi + return 0 +} + + +# zed_notify_pushbullet (subject, pathname) +# +# Send a notification via Pushbullet <https://www.pushbullet.com/>. +# The access token (ZED_PUSHBULLET_ACCESS_TOKEN) identifies this client to the +# Pushbullet server. The optional channel tag (ZED_PUSHBULLET_CHANNEL_TAG) is +# for pushing to notification feeds that can be subscribed to; if a channel is +# not defined, push notifications will instead be sent to all devices +# associated with the account specified by the access token. +# +# Requires awk, curl, and sed executables to be installed in the standard PATH. +# +# References +# https://docs.pushbullet.com/ +# https://www.pushbullet.com/security +# +# Arguments +# subject: notification subject +# pathname: pathname containing the notification message (OPTIONAL) +# +# Globals +# ZED_PUSHBULLET_ACCESS_TOKEN +# ZED_PUSHBULLET_CHANNEL_TAG +# +# Return +# 0: notification sent +# 1: notification failed +# 2: not configured +# +zed_notify_pushbullet() +{ + local subject="$1" + local pathname="${2:-"/dev/null"}" + local msg_body + local msg_tag + local msg_json + local msg_out + local msg_err + local url="https://api.pushbullet.com/v2/pushes" + + [ -n "${ZED_PUSHBULLET_ACCESS_TOKEN}" ] || return 2 + + [ -n "${subject}" ] || return 1 + if [ ! -r "${pathname}" ]; then + zed_log_err "pushbullet cannot read \"${pathname}\"" + return 1 + fi + + zed_check_cmd "awk" "curl" "sed" || return 1 + + # Escape the following characters in the message body for JSON: + # newline, backslash, double quote, horizontal tab, vertical tab, + # and carriage return. + # + msg_body="$(awk '{ ORS="\\n" } { gsub(/\\/, "\\\\"); gsub(/"/, "\\\""); + gsub(/\t/, "\\t"); gsub(/\f/, "\\f"); gsub(/\r/, "\\r"); print }' \ + "${pathname}")" + + # Push to a channel if one is configured. + # + [ -n "${ZED_PUSHBULLET_CHANNEL_TAG}" ] && msg_tag="$(printf \ + '"channel_tag": "%s", ' "${ZED_PUSHBULLET_CHANNEL_TAG}")" + + # Construct the JSON message for pushing a note. + # + msg_json="$(printf '{%s"type": "note", "title": "%s", "body": "%s"}' \ + "${msg_tag}" "${subject}" "${msg_body}")" + + # Send the POST request and check for errors. + # + msg_out="$(curl -u "${ZED_PUSHBULLET_ACCESS_TOKEN}:" -X POST "${url}" \ + --header "Content-Type: application/json" --data-binary "${msg_json}" \ + 2>/dev/null)"; rv=$? + if [ "${rv}" -ne 0 ]; then + zed_log_err "curl exit=${rv}" + return 1 + fi + msg_err="$(echo "${msg_out}" \ + | sed -n -e 's/.*"error" *:.*"message" *: *"\([^"]*\)".*/\1/p')" + if [ -n "${msg_err}" ]; then + zed_log_err "pushbullet \"${msg_err}"\" + return 1 + fi + return 0 +} + + +# zed_notify_slack_webhook (subject, pathname) +# +# Notification via Slack Webhook <https://api.slack.com/incoming-webhooks>. +# The Webhook URL (ZED_SLACK_WEBHOOK_URL) identifies this client to the +# Slack channel. +# +# Requires awk, curl, and sed executables to be installed in the standard PATH. +# +# References +# https://api.slack.com/incoming-webhooks +# +# Arguments +# subject: notification subject +# pathname: pathname containing the notification message (OPTIONAL) +# +# Globals +# ZED_SLACK_WEBHOOK_URL +# +# Return +# 0: notification sent +# 1: notification failed +# 2: not configured +# +zed_notify_slack_webhook() +{ + [ -n "${ZED_SLACK_WEBHOOK_URL}" ] || return 2 + + local subject="$1" + local pathname="${2:-"/dev/null"}" + local msg_body + local msg_tag + local msg_json + local msg_out + local msg_err + local url="${ZED_SLACK_WEBHOOK_URL}" + + [ -n "${subject}" ] || return 1 + if [ ! -r "${pathname}" ]; then + zed_log_err "slack webhook cannot read \"${pathname}\"" + return 1 + fi + + zed_check_cmd "awk" "curl" "sed" || return 1 + + # Escape the following characters in the message body for JSON: + # newline, backslash, double quote, horizontal tab, vertical tab, + # and carriage return. + # + msg_body="$(awk '{ ORS="\\n" } { gsub(/\\/, "\\\\"); gsub(/"/, "\\\""); + gsub(/\t/, "\\t"); gsub(/\f/, "\\f"); gsub(/\r/, "\\r"); print }' \ + "${pathname}")" + + # Construct the JSON message for posting. + # + msg_json="$(printf '{"text": "*%s*\\n%s"}' "${subject}" "${msg_body}" )" + + # Send the POST request and check for errors. + # + msg_out="$(curl -X POST "${url}" \ + --header "Content-Type: application/json" --data-binary "${msg_json}" \ + 2>/dev/null)"; rv=$? + if [ "${rv}" -ne 0 ]; then + zed_log_err "curl exit=${rv}" + return 1 + fi + msg_err="$(echo "${msg_out}" \ + | sed -n -e 's/.*"error" *:.*"message" *: *"\([^"]*\)".*/\1/p')" + if [ -n "${msg_err}" ]; then + zed_log_err "slack webhook \"${msg_err}"\" + return 1 + fi + return 0 +} + +# zed_notify_pushover (subject, pathname) +# +# Send a notification via Pushover <https://pushover.net/>. +# The access token (ZED_PUSHOVER_TOKEN) identifies this client to the +# Pushover server. The user token (ZED_PUSHOVER_USER) defines the user or +# group to which the notification will be sent. +# +# Requires curl and sed executables to be installed in the standard PATH. +# +# References +# https://pushover.net/api +# +# Arguments +# subject: notification subject +# pathname: pathname containing the notification message (OPTIONAL) +# +# Globals +# ZED_PUSHOVER_TOKEN +# ZED_PUSHOVER_USER +# +# Return +# 0: notification sent +# 1: notification failed +# 2: not configured +# +zed_notify_pushover() +{ + local subject="$1" + local pathname="${2:-"/dev/null"}" + local msg_body + local msg_out + local msg_err + local url="https://api.pushover.net/1/messages.json" + + [ -n "${ZED_PUSHOVER_TOKEN}" ] && [ -n "${ZED_PUSHOVER_USER}" ] || return 2 + + if [ ! -r "${pathname}" ]; then + zed_log_err "pushover cannot read \"${pathname}\"" + return 1 + fi + + zed_check_cmd "curl" "sed" || return 1 + + # Read the message body in. + # + msg_body="$(cat "${pathname}")" + + if [ -z "${msg_body}" ] + then + msg_body=$subject + subject="" + fi + + # Send the POST request and check for errors. + # + msg_out="$( \ + curl \ + --form-string "token=${ZED_PUSHOVER_TOKEN}" \ + --form-string "user=${ZED_PUSHOVER_USER}" \ + --form-string "message=${msg_body}" \ + --form-string "title=${subject}" \ + "${url}" \ + 2>/dev/null \ + )"; rv=$? + if [ "${rv}" -ne 0 ]; then + zed_log_err "curl exit=${rv}" + return 1 + fi + msg_err="$(echo "${msg_out}" \ + | sed -n -e 's/.*"errors" *:.*\[\(.*\)\].*/\1/p')" + if [ -n "${msg_err}" ]; then + zed_log_err "pushover \"${msg_err}"\" + return 1 + fi + return 0 +} + + +# zed_notify_ntfy (subject, pathname) +# +# Send a notification via Ntfy.sh <https://ntfy.sh/>. +# The ntfy topic (ZED_NTFY_TOPIC) identifies the topic that the notification +# will be sent to Ntfy.sh server. The ntfy url (ZED_NTFY_URL) defines the +# self-hosted or provided hosted ntfy service location. The ntfy access token +# <https://docs.ntfy.sh/publish/#access-tokens> (ZED_NTFY_ACCESS_TOKEN) reprsents an +# access token that could be used if a topic is read/write protected. If a +# topic can be written to publicaly, a ZED_NTFY_ACCESS_TOKEN is not required. +# +# Requires curl and sed executables to be installed in the standard PATH. +# +# References +# https://docs.ntfy.sh +# +# Arguments +# subject: notification subject +# pathname: pathname containing the notification message (OPTIONAL) +# +# Globals +# ZED_NTFY_TOPIC +# ZED_NTFY_ACCESS_TOKEN (OPTIONAL) +# ZED_NTFY_URL +# +# Return +# 0: notification sent +# 1: notification failed +# 2: not configured +# +zed_notify_ntfy() +{ + local subject="$1" + local pathname="${2:-"/dev/null"}" + local msg_body + local msg_out + local msg_err + + [ -n "${ZED_NTFY_TOPIC}" ] || return 2 + local url="${ZED_NTFY_URL:-"https://ntfy.sh"}/${ZED_NTFY_TOPIC}" + + if [ ! -r "${pathname}" ]; then + zed_log_err "ntfy cannot read \"${pathname}\"" + return 1 + fi + + zed_check_cmd "curl" "sed" || return 1 + + # Read the message body in. + # + msg_body="$(cat "${pathname}")" + + if [ -z "${msg_body}" ] + then + msg_body=$subject + subject="" + fi + + # Send the POST request and check for errors. + # + if [ -n "${ZED_NTFY_ACCESS_TOKEN}" ]; then + msg_out="$( \ + curl \ + -u ":${ZED_NTFY_ACCESS_TOKEN}" \ + -H "Title: ${subject}" \ + -d "${msg_body}" \ + -H "Priority: high" \ + "${url}" \ + 2>/dev/null \ + )"; rv=$? + else + msg_out="$( \ + curl \ + -H "Title: ${subject}" \ + -d "${msg_body}" \ + -H "Priority: high" \ + "${url}" \ + 2>/dev/null \ + )"; rv=$? + fi + if [ "${rv}" -ne 0 ]; then + zed_log_err "curl exit=${rv}" + return 1 + fi + msg_err="$(echo "${msg_out}" \ + | sed -n -e 's/.*"errors" *:.*\[\(.*\)\].*/\1/p')" + if [ -n "${msg_err}" ]; then + zed_log_err "ntfy \"${msg_err}"\" + return 1 + fi + return 0 +} + + +# zed_notify_gotify (subject, pathname) +# +# Send a notification via Gotify <https://gotify.net/>. +# The Gotify URL (ZED_GOTIFY_URL) defines a self-hosted Gotify location. +# The Gotify application token (ZED_GOTIFY_APPTOKEN) defines a +# Gotify application token which is associated with a message. +# The optional Gotify priority value (ZED_GOTIFY_PRIORITY) overrides the +# default or configured priority at the Gotify server for the application. +# +# Requires curl and sed executables to be installed in the standard PATH. +# +# References +# https://gotify.net/docs/index +# +# Arguments +# subject: notification subject +# pathname: pathname containing the notification message (OPTIONAL) +# +# Globals +# ZED_GOTIFY_URL +# ZED_GOTIFY_APPTOKEN +# ZED_GOTIFY_PRIORITY +# +# Return +# 0: notification sent +# 1: notification failed +# 2: not configured +# +zed_notify_gotify() +{ + local subject="$1" + local pathname="${2:-"/dev/null"}" + local msg_body + local msg_out + local msg_err + + [ -n "${ZED_GOTIFY_URL}" ] && [ -n "${ZED_GOTIFY_APPTOKEN}" ] || return 2 + local url="${ZED_GOTIFY_URL}/message?token=${ZED_GOTIFY_APPTOKEN}" + + if [ ! -r "${pathname}" ]; then + zed_log_err "gotify cannot read \"${pathname}\"" + return 1 + fi + + zed_check_cmd "curl" "sed" || return 1 + + # Read the message body in. + # + msg_body="$(cat "${pathname}")" + + if [ -z "${msg_body}" ] + then + msg_body=$subject + subject="" + fi + + # Send the POST request and check for errors. + # + if [ -n "${ZED_GOTIFY_PRIORITY}" ]; then + msg_out="$( \ + curl \ + --form-string "title=${subject}" \ + --form-string "message=${msg_body}" \ + --form-string "priority=${ZED_GOTIFY_PRIORITY}" \ + "${url}" \ + 2>/dev/null \ + )"; rv=$? + else + msg_out="$( \ + curl \ + --form-string "title=${subject}" \ + --form-string "message=${msg_body}" \ + "${url}" \ + 2>/dev/null \ + )"; rv=$? + fi + + if [ "${rv}" -ne 0 ]; then + zed_log_err "curl exit=${rv}" + return 1 + fi + msg_err="$(echo "${msg_out}" \ + | sed -n -e 's/.*"errors" *:.*\[\(.*\)\].*/\1/p')" + if [ -n "${msg_err}" ]; then + zed_log_err "gotify \"${msg_err}"\" + return 1 + fi + return 0 +} + + + +# zed_rate_limit (tag, [interval]) +# +# Check whether an event of a given type [tag] has already occurred within the +# last [interval] seconds. +# +# This function obtains a lock on the statefile using file descriptor 9. +# +# Arguments +# tag: arbitrary string for grouping related events to rate-limit +# interval: time interval in seconds (OPTIONAL) +# +# Globals +# ZED_NOTIFY_INTERVAL_SECS +# ZED_RUNDIR +# +# Return +# 0 if the event should be processed +# 1 if the event should be dropped +# +# State File Format +# time;tag +# +zed_rate_limit() +{ + local tag="$1" + local interval="${2:-${ZED_NOTIFY_INTERVAL_SECS}}" + local lockfile="zed.zedlet.state.lock" + local lockfile_fd=9 + local statefile="${ZED_RUNDIR}/zed.zedlet.state" + local time_now + local time_prev + local umask_bak + local rv=0 + + [ -n "${tag}" ] || return 0 + + zed_lock "${lockfile}" "${lockfile_fd}" + time_now="$(date +%s)" + time_prev="$(grep -E "^[0-9]+;${tag}\$" "${statefile}" 2>/dev/null \ + | tail -1 | cut -d\; -f1)" + + if [ -n "${time_prev}" ] \ + && [ "$((time_now - time_prev))" -lt "${interval}" ]; then + rv=1 + else + umask_bak="$(umask)" + umask 077 + grep -E -v "^[0-9]+;${tag}\$" "${statefile}" 2>/dev/null \ + > "${statefile}.$$" + echo "${time_now};${tag}" >> "${statefile}.$$" + mv -f "${statefile}.$$" "${statefile}" + umask "${umask_bak}" + fi + + zed_unlock "${lockfile}" "${lockfile_fd}" + return "${rv}" +} + + +# zed_guid_to_pool (guid) +# +# Convert a pool GUID into its pool name (like "tank") +# Arguments +# guid: pool GUID (decimal or hex) +# +# Return +# Pool name +# +zed_guid_to_pool() +{ + if [ -z "$1" ] ; then + return + fi + + guid="$(printf "%u" "$1")" + $ZPOOL get -H -ovalue,name guid | awk '$1 == '"$guid"' {print $2; exit}' +} + +# zed_exit_if_ignoring_this_event +# +# Exit the script if we should ignore this event, as determined by +# $ZED_SYSLOG_SUBCLASS_INCLUDE and $ZED_SYSLOG_SUBCLASS_EXCLUDE in zed.rc. +# This function assumes you've imported the normal zed variables. +zed_exit_if_ignoring_this_event() +{ + if [ -n "${ZED_SYSLOG_SUBCLASS_INCLUDE}" ]; then + eval "case ${ZEVENT_SUBCLASS} in + ${ZED_SYSLOG_SUBCLASS_INCLUDE});; + *) exit 0;; + esac" + elif [ -n "${ZED_SYSLOG_SUBCLASS_EXCLUDE}" ]; then + eval "case ${ZEVENT_SUBCLASS} in + ${ZED_SYSLOG_SUBCLASS_EXCLUDE}) exit 0;; + *);; + esac" + fi +} diff --git a/straper/db/public/zfs/etc-zfs/zed.d/zed.rc b/straper/db/public/zfs/etc-zfs/zed.d/zed.rc new file mode 100644 index 0000000..47fb016 --- /dev/null +++ b/straper/db/public/zfs/etc-zfs/zed.d/zed.rc @@ -0,0 +1,199 @@ +## +# zed.rc – ZEDLET configuration. +## +# shellcheck disable=SC2034 + +## +# Absolute path to the debug output file. +# +#ZED_DEBUG_LOG="/tmp/zed.debug.log" + +## +# Email address of the zpool administrator for receipt of notifications; +# multiple addresses can be specified if they are delimited by whitespace. +# Email will only be sent if ZED_EMAIL_ADDR is defined. +# Enabled by default; comment to disable. +# +ZED_EMAIL_ADDR="root" + +## +# Name or path of executable responsible for sending notifications via email; +# the mail program must be capable of reading a message body from stdin. +# Email will only be sent if ZED_EMAIL_ADDR is defined. +# +#ZED_EMAIL_PROG="mail" + +## +# Command-line options for ZED_EMAIL_PROG. +# The string @ADDRESS@ will be replaced with the recipient email address(es). +# The string @SUBJECT@ will be replaced with the notification subject; +# this should be protected with quotes to prevent word-splitting. +# Email will only be sent if ZED_EMAIL_ADDR is defined. +# If @SUBJECT@ was omited here, a "Subject: ..." header will be added to notification +# +#ZED_EMAIL_OPTS="-s '@SUBJECT@' @ADDRESS@" + +## +# Default directory for zed lock files. +# +#ZED_LOCKDIR="/var/lock" + +## +# Minimum number of seconds between notifications for a similar event. +# +ZED_NOTIFY_INTERVAL_SECS=3600 + +## +# Notification verbosity. +# If set to 0, suppress notification if the pool is healthy. +# If set to 1, send notification regardless of pool health. +# +#ZED_NOTIFY_VERBOSE=0 + +## +# Send notifications for 'ereport.fs.zfs.data' events. +# Disabled by default, any non-empty value will enable the feature. +# +#ZED_NOTIFY_DATA= + +## +# Pushbullet access token. +# This grants full access to your account -- protect it accordingly! +# <https://www.pushbullet.com/get-started> +# <https://www.pushbullet.com/account> +# Disabled by default; uncomment to enable. +# +#ZED_PUSHBULLET_ACCESS_TOKEN="" + +## +# Pushbullet channel tag for push notification feeds that can be subscribed to. +# <https://www.pushbullet.com/my-channel> +# If not defined, push notifications will instead be sent to all devices +# associated with the account specified by the access token. +# Disabled by default; uncomment to enable. +# +#ZED_PUSHBULLET_CHANNEL_TAG="" + +## +# Slack Webhook URL. +# This allows posting to the given channel and includes an access token. +# <https://api.slack.com/incoming-webhooks> +# Disabled by default; uncomment to enable. +# +#ZED_SLACK_WEBHOOK_URL="" + +## +# Pushover token. +# This defines the application from which the notification will be sent. +# <https://pushover.net/api#registration> +# Disabled by default; uncomment to enable. +# ZED_PUSHOVER_USER, below, must also be configured. +# +#ZED_PUSHOVER_TOKEN="" + +## +# Pushover user key. +# This defines which user or group will receive Pushover notifications. +# <https://pushover.net/api#identifiers> +# Disabled by default; uncomment to enable. +# ZED_PUSHOVER_TOKEN, above, must also be configured. +#ZED_PUSHOVER_USER="" + +## +# Default directory for zed state files. +# +#ZED_RUNDIR="/var/run" + +## +# Turn on/off enclosure LEDs when drives get DEGRADED/FAULTED. This works for +# device mapper and multipath devices as well. This works with JBOD enclosures +# and NVMe PCI drives (assuming they're supported by Linux in sysfs). +# +ZED_USE_ENCLOSURE_LEDS=1 + +## +# Run a scrub after every resilver +# Disabled by default, 1 to enable and 0 to disable. +#ZED_SCRUB_AFTER_RESILVER=0 + +## +# The syslog priority (e.g., specified as a "facility.level" pair). +# +#ZED_SYSLOG_PRIORITY="daemon.notice" + +## +# The syslog tag for marking zed events. +# +#ZED_SYSLOG_TAG="zed" + +## +# Which set of event subclasses to log +# By default, events from all subclasses are logged. +# If ZED_SYSLOG_SUBCLASS_INCLUDE is set, only subclasses +# matching the pattern are logged. Use the pipe symbol (|) +# or shell wildcards (*, ?) to match multiple subclasses. +# Otherwise, if ZED_SYSLOG_SUBCLASS_EXCLUDE is set, the +# matching subclasses are excluded from logging. +#ZED_SYSLOG_SUBCLASS_INCLUDE="checksum|scrub_*|vdev.*" +ZED_SYSLOG_SUBCLASS_EXCLUDE="history_event" + +## +# Use GUIDs instead of names when logging pool and vdevs +# Disabled by default, 1 to enable and 0 to disable. +#ZED_SYSLOG_DISPLAY_GUIDS=1 + +## +# Power off the drive's slot in the enclosure if it becomes FAULTED. This can +# help silence misbehaving drives. This assumes your drive enclosure fully +# supports slot power control via sysfs. +#ZED_POWER_OFF_ENCLOSURE_SLOT_ON_FAULT=1 + +## +# Power off the drive's slot in the enclosure if there is a hung I/O which +# exceeds the deadman timeout. This can help prevent a single misbehaving +# drive from rendering a redundant pool unavailable. This assumes your drive +# enclosure fully supports slot power control via sysfs. +#ZED_POWER_OFF_ENCLOSURE_SLOT_ON_DEADMAN=1 + +## +# Ntfy topic +# This defines which topic will receive the ntfy notification. +# <https://docs.ntfy.sh/publish/> +# Disabled by default; uncomment to enable. +#ZED_NTFY_TOPIC="" + +## +# Ntfy access token (optional for public topics) +# This defines an access token which can be used +# to allow you to authenticate when sending to topics +# <https://docs.ntfy.sh/publish/#access-tokens> +# Disabled by default; uncomment to enable. +#ZED_NTFY_ACCESS_TOKEN="" + +## +# Ntfy Service URL +# This defines which service the ntfy call will be directed toward +# <https://docs.ntfy.sh/install/> +# https://ntfy.sh by default; uncomment to enable an alternative service url. +#ZED_NTFY_URL="https://ntfy.sh" + +## +# Gotify server URL +# This defines a URL that the Gotify call will be directed toward. +# <https://gotify.net/docs/index> +# Disabled by default; uncomment to enable. +#ZED_GOTIFY_URL="" + +## +# Gotify application token +# This defines a Gotify application token which a message is associated with. +# This token is generated when an application is created on the Gotify server. +# Disabled by default; uncomment to enable. +#ZED_GOTIFY_APPTOKEN="" + +## +# Gotify priority (optional) +# If defined, this overrides the default priority of the +# Gotify application associated with ZED_GOTIFY_APPTOKEN. +# Value is an integer 0 and up. +#ZED_GOTIFY_PRIORITY="" diff --git a/straper/db/public/zfs/etc-zfs/zfs-functions b/straper/db/public/zfs/etc-zfs/zfs-functions new file mode 100644 index 0000000..8c30894 --- /dev/null +++ b/straper/db/public/zfs/etc-zfs/zfs-functions @@ -0,0 +1,434 @@ +# SPDX-License-Identifier: BSD-2-Clause +# This is a script with common functions etc used by zfs-import, zfs-load-key, +# zfs-mount, zfs-share and zfs-zed. +# +# It is _NOT_ to be called independently +# +# Released under the 2-clause BSD license. +# +# This script is based on debian/zfsutils.zfs.init from the +# Debian GNU/kFreeBSD zfsutils 8.1-3 package, written by Aurelien Jarno. + +PATH=/sbin:/bin:/usr/bin:/usr/sbin + +# Source function library +if [ -f /etc/rc.d/init.d/functions ]; then + # RedHat and derivatives + . /etc/rc.d/init.d/functions +elif [ -L /etc/init.d/functions.sh ]; then + # Gentoo + . /etc/init.d/functions.sh +elif [ -f /lib/lsb/init-functions ]; then + # LSB, Debian, and derivatives + . /lib/lsb/init-functions +fi + +# Of course the functions we need are called differently +# on different distributions - it would be way too easy +# otherwise!! +if type log_failure_msg > /dev/null 2>&1 ; then + # LSB functions - fall through + zfs_log_begin_msg() { log_begin_msg "$1"; } + zfs_log_end_msg() { log_end_msg "$1"; } + zfs_log_failure_msg() { log_failure_msg "$1"; } + zfs_log_progress_msg() { log_progress_msg "$1"; } +elif type success > /dev/null 2>&1 ; then + # Fedora/RedHat functions + zfs_set_ifs() { + # For some reason, the init function library have a problem + # with a changed IFS, so this function goes around that. + local tIFS="$1" + if [ -n "$tIFS" ] + then + TMP_IFS="$IFS" + IFS="$tIFS" + fi + } + + zfs_log_begin_msg() { printf "%s" "$1 "; } + zfs_log_end_msg() { + # shellcheck disable=SC2154 + zfs_set_ifs "$OLD_IFS" + if [ "$1" -eq 0 ]; then + success + else + failure + fi + echo + zfs_set_ifs "$TMP_IFS" + } + zfs_log_failure_msg() { + zfs_set_ifs "$OLD_IFS" + failure + echo + zfs_set_ifs "$TMP_IFS" + } + zfs_log_progress_msg() { printf "%s" "$""$1"; } +elif type einfo > /dev/null 2>&1 ; then + # Gentoo functions + zfs_log_begin_msg() { ebegin "$1"; } + zfs_log_end_msg() { eend "$1"; } + zfs_log_failure_msg() { eend "$1"; } +# zfs_log_progress_msg() { printf "%s" "$1"; } + zfs_log_progress_msg() { :; } +else + # Unknown - simple substitutes. + zfs_log_begin_msg() { printf "%s" "$1"; } + zfs_log_end_msg() { + ret=$1 + if [ "$ret" -ge 1 ]; then + echo " failed!" + else + echo " success" + fi + return "$ret" + } + zfs_log_failure_msg() { echo "$1"; } + zfs_log_progress_msg() { printf "%s" "$1"; } +fi + +# Paths to what we need +ZFS="/usr/sbin/zfs" +ZED="/usr/sbin/zed" +ZPOOL="/usr/sbin/zpool" +ZPOOL_CACHE="/etc/zfs/zpool.cache" + +# Sensible defaults +ZFS_LOAD_KEY='yes' +ZFS_UNLOAD_KEY='no' +ZFS_MOUNT='yes' +ZFS_UNMOUNT='yes' +ZFS_SHARE='yes' +ZFS_UNSHARE='yes' + +# Source zfs configuration, overriding the defaults +if [ -f /etc/default/zfs ]; then + . /etc/default/zfs +fi + +# ---------------------------------------------------- + +export ZFS ZED ZPOOL ZPOOL_CACHE ZFS_LOAD_KEY ZFS_UNLOAD_KEY ZFS_MOUNT ZFS_UNMOUNT \ + ZFS_SHARE ZFS_UNSHARE + +zfs_action() +{ + local MSG="$1"; shift + local CMD="$*" + local ret + + zfs_log_begin_msg "$MSG " + $CMD + ret=$? + if [ "$ret" -eq 0 ]; then + zfs_log_end_msg "$ret" + else + zfs_log_failure_msg "$ret" + fi + + return "$ret" +} + +# Returns +# 0 if daemon has been started +# 1 if daemon was already running +# 2 if daemon could not be started +# 3 if unsupported +# +zfs_daemon_start() +{ + local PIDFILE="$1"; shift + local DAEMON_BIN="$1"; shift + + if type start-stop-daemon > /dev/null 2>&1 ; then + # LSB functions + start-stop-daemon --start --quiet --pidfile "$PIDFILE" \ + --exec "$DAEMON_BIN" --test > /dev/null || return 1 + + # shellcheck disable=SC2086 + start-stop-daemon --start --quiet --exec "$DAEMON_BIN" -- \ + "$@" || return 2 + + # On Debian, there's a 'sendsigs' script that will + # kill basically everything quite early and zed is stopped + # much later than that. We don't want zed to be among them, + # so add the zed pid to list of pids to ignore. + if [ -f "$PIDFILE" ] && [ -d /run/sendsigs.omit.d ] + then + ln -sf "$PIDFILE" /run/sendsigs.omit.d/zed + fi + elif type daemon > /dev/null 2>&1 ; then + # Fedora/RedHat functions + # shellcheck disable=SC2086 + daemon --pidfile "$PIDFILE" "$DAEMON_BIN" "$@" + return $? + else + # Unsupported + return 3 + fi + + return 0 +} + +# Returns +# 0 if daemon has been stopped +# 1 if daemon was already stopped +# 2 if daemon could not be stopped +# 3 if unsupported +# +zfs_daemon_stop() +{ + local PIDFILE="$1" + local DAEMON_BIN="$2" + local DAEMON_NAME="$3" + + if type start-stop-daemon > /dev/null 2>&1 ; then + # LSB functions + start-stop-daemon --stop --quiet --retry=TERM/30/KILL/5 \ + --pidfile "$PIDFILE" --name "$DAEMON_NAME" + ret="$?" + [ "$ret" = 0 ] && rm -f "$PIDFILE" + + return "$ret" + elif type killproc > /dev/null 2>&1 ; then + # Fedora/RedHat functions + killproc -p "$PIDFILE" "$DAEMON_NAME" + ret="$?" + [ "$ret" = 0 ] && rm -f "$PIDFILE" + + return "$ret" + else + # Unsupported + return 3 + fi + + return 0 +} + +# Returns status +zfs_daemon_status() +{ + local PIDFILE="$1" + local DAEMON_BIN="$2" + local DAEMON_NAME="$3" + + if type status_of_proc > /dev/null 2>&1 ; then + # LSB functions + status_of_proc "$DAEMON_NAME" "$DAEMON_BIN" + return $? + elif type status > /dev/null 2>&1 ; then + # Fedora/RedHat functions + status -p "$PIDFILE" "$DAEMON_NAME" + return $? + else + # Unsupported + return 3 + fi + + return 0 +} + +zfs_daemon_reload() +{ + local PIDFILE="$1" + local DAEMON_NAME="$2" + + if type start-stop-daemon > /dev/null 2>&1 ; then + # LSB functions + start-stop-daemon --stop --signal 1 --quiet \ + --pidfile "$PIDFILE" --name "$DAEMON_NAME" + return $? + elif type killproc > /dev/null 2>&1 ; then + # Fedora/RedHat functions + killproc -p "$PIDFILE" "$DAEMON_NAME" -HUP + return $? + else + # Unsupported + return 3 + fi + + return 0 +} + +zfs_installed() +{ + if [ ! -x "$ZPOOL" ]; then + return 1 + else + # Test if it works (will catch missing/broken libs etc) + "$ZPOOL" -? > /dev/null 2>&1 + return $? + fi + + if [ ! -x "$ZFS" ]; then + return 2 + else + # Test if it works (will catch missing/broken libs etc) + "$ZFS" -? > /dev/null 2>&1 + return $? + fi + + return 0 +} + +# Trigger udev and wait for it to settle. +udev_trigger() +{ + if [ -x /sbin/udevadm ]; then + /sbin/udevadm trigger --action=change --subsystem-match=block + /sbin/udevadm settle + elif [ -x /sbin/udevsettle ]; then + /sbin/udevtrigger + /sbin/udevsettle + fi +} + +# Do a lot of checks to make sure it's 'safe' to continue with the import. +checksystem() +{ + if grep -qiE '(^|[^\\](\\\\)* )zfs=(off|no|0)( |$)' /proc/cmdline; + then + # Called with zfs=(off|no|0) - bail because we don't + # want anything import, mounted or shared. + # HOWEVER, only do this if we're called at the boot up + # (from init), not if we're running interactively (as in + # from the shell - we know what we're doing). + # shellcheck disable=SC2154 + [ -n "$init" ] && exit 3 + fi + + # Check if ZFS is installed. + zfs_installed || return 5 + + # Just make sure that /dev/zfs is created. + udev_trigger + + return 0 +} + +get_root_pool() +{ + # shellcheck disable=SC2046 + set -- $(mount | grep ' on / ') + [ "$5" = "zfs" ] && echo "${1%%/*}" +} + +# Check if a variable is 'yes' (any case) or '1' +# Returns TRUE if set. +check_boolean() +{ + local var="$1" + + echo "$var" | grep -Eiq "^yes$|^on$|^true$|^1$" && return 0 || return 1 +} + +check_module_loaded() +{ + module="$1" + + [ -r "/sys/module/${module}/version" ] && return 0 || return 1 +} + +load_module() +{ + module="$1" + + # Load the zfs module stack + if ! check_module_loaded "$module"; then + if ! /sbin/modprobe "$module"; then + return 5 + fi + fi + return 0 +} + +# first parameter is a regular expression that filters mtab +read_mtab() +{ + local match="$1" + local fs mntpnt fstype opts rest + + # Unset all MTAB_* variables + # shellcheck disable=SC2046 + unset $(env | sed -e '/^MTAB_/!d' -e 's,=.*,,') + + while read -r fs mntpnt fstype opts rest; do + if echo "$fs $mntpnt $fstype $opts" | grep -qE "$match"; then + # * Fix problems (!?) in the mounts file. It will record + # 'rpool 1' as 'rpool\0401' instead of 'rpool\00401' + # which seems to be the correct (at least as far as + # 'printf' is concerned). + # * We need to use the external echo, because the + # internal one would interpret the backslash code + # (incorrectly), giving us a instead. + mntpnt=$(/bin/echo "$mntpnt" | sed 's,\\0,\\00,g') + fs=$(/bin/echo "$fs" | sed 's,\\0,\\00,') + + # Remove 'unwanted' characters. + mntpnt=$(printf '%b' "$mntpnt" | tr -d '/. -') + fs=$(printf '%b' "$fs") + + # Set the variable. + eval export "MTAB_$mntpnt=\"$fs\"" + fi + done < /proc/self/mounts +} + +in_mtab() +{ + local mntpnt="$1" + # Remove 'unwanted' characters. + mntpnt=$(printf '%b' "$mntpnt" | tr -d '/. -') + local var + + var="$(eval echo "MTAB_$mntpnt")" + [ "$(eval echo "$""$var")" != "" ] + return "$?" +} + +# first parameter is a regular expression that filters fstab +read_fstab() +{ + local match="$1" + local i var + + # Unset all FSTAB_* variables + # shellcheck disable=SC2046 + unset $(env | sed -e '/^FSTAB_/!d' -e 's,=.*,,') + + i=0 + while read -r fs mntpnt fstype opts; do + echo "$fs" | grep -qE '^#|^$' && continue + echo "$mntpnt" | grep -qE '^none|^swap' && continue + echo "$fstype" | grep -qE '^swap' && continue + + if echo "$fs $mntpnt $fstype $opts" | grep -qE "$match"; then + eval export "FSTAB_dev_$i=$fs" + fs=$(printf '%b' "$fs" | tr '/' '_') + eval export "FSTAB_$i=$mntpnt" + + i=$((i + 1)) + fi + done < /etc/fstab +} + +in_fstab() +{ + local var + + var="$(eval echo "FSTAB_$1")" + [ "${var}" != "" ] + return $? +} + +is_mounted() +{ + local mntpt="$1" + local mp + + while read -r _ mp _; do + [ "$mp" = "$mntpt" ] && return 0 + done < /proc/self/mounts + + return 1 +} diff --git a/straper/db/public/zfs/etc-zfs/zpool.cache b/straper/db/public/zfs/etc-zfs/zpool.cache Binary files differnew file mode 100644 index 0000000..7557795 --- /dev/null +++ b/straper/db/public/zfs/etc-zfs/zpool.cache diff --git a/straper/db/public/zfs/etc-zfs/zpool.d/ata_err b/straper/db/public/zfs/etc-zfs/zpool.d/ata_err new file mode 100755 index 0000000..8ad3e10 --- /dev/null +++ b/straper/db/public/zfs/etc-zfs/zpool.d/ata_err @@ -0,0 +1,249 @@ +#!/bin/sh +# +# Show SMART stats +# + +helpstr=" +smart: Show SMART temperature and error stats (specific to drive type) +smartx: Show SMART extended drive stats (specific to drive type). +temp: Show SMART drive temperature in celsius (all drives). +health: Show reported SMART status (all drives). +r_proc: Show SMART read GBytes processed over drive lifetime (SAS). +w_proc: Show SMART write GBytes processed over drive lifetime (SAS). +r_ucor: Show SMART read uncorrectable errors (SAS). +w_ucor: Show SMART write uncorrectable errors (SAS). +nonmed: Show SMART non-medium errors (SAS). +defect: Show SMART grown defect list (SAS). +hours_on: Show number of hours drive powered on (all drives). +realloc: Show SMART reallocated sectors count (ATA). +rep_ucor: Show SMART reported uncorrectable count (ATA). +cmd_to: Show SMART command timeout count (ATA). +pend_sec: Show SMART current pending sector count (ATA). +off_ucor: Show SMART offline uncorrectable errors (ATA). +ata_err: Show SMART ATA errors (ATA). +pwr_cyc: Show SMART power cycle count (ATA). +serial: Show disk serial number. +nvme_err: Show SMART NVMe errors (NVMe). +smart_test: Show SMART self-test results summary. +test_type: Show SMART self-test type (short, long... ). +test_status: Show SMART self-test status. +test_progress: Show SMART self-test percentage done. +test_ended: Show when the last SMART self-test ended (if supported). +" + +# Hack for developer testing +# +# If you set $samples to a directory containing smartctl output text files, +# we will use them instead of running smartctl on the vdevs. This can be +# useful if you want to test a bunch of different smartctl outputs. Also, if +# $samples is set, and additional 'file' column is added to the zpool output +# showing the filename. +samples= + +# get_filename_from_dir DIR +# +# Look in directory DIR and return a filename from it. The filename returned +# is chosen quasi-sequentially (based off our PID). This allows us to return +# a different filename every time this script is invoked (which we do for each +# vdev), without having to maintain state. +get_filename_from_dir() +{ + dir=$1 + pid="$$" + num_files=$(find "$dir" -maxdepth 1 -type f | wc -l) + mod=$((pid % num_files)) + i=0 + find "$dir" -type f -printf '%f\n' | while read -r file ; do + if [ "$mod" = "$i" ] ; then + echo "$file" + break + fi + i=$((i+1)) + done +} + +script="${0##*/}" + +if [ "$1" = "-h" ] ; then + echo "$helpstr" | grep "$script:" | tr -s '\t' | cut -f 2- + exit +fi + +# Sometimes, UPATH ends up /dev/(null). +# That should be corrected, but for now... +# shellcheck disable=SC2154 +if [ ! -b "$VDEV_UPATH" ]; then + somepath="${VDEV_PATH}" +else + somepath="${VDEV_UPATH}" +fi + +if [ -b "$somepath" ] && PATH="/usr/sbin:$PATH" command -v smartctl > /dev/null || [ -n "$samples" ] ; then + if [ -n "$samples" ] ; then + # cat a smartctl output text file instead of running smartctl + # on a vdev (only used for developer testing). + file=$(get_filename_from_dir "$samples") + echo "file=$file" + raw_out=$(cat "$samples/$file") + else + raw_out=$(sudo smartctl -a "$somepath") + fi + + # What kind of drive are we? Look for the right line in smartctl: + # + # SAS: + # Transport protocol: SAS + # + # SATA: + # ATA Version is: 8 + # + # NVMe: + # SMART/Health Information (NVMe Log 0xnn, NSID 0xnn) + # + out=$(echo "$raw_out" | awk ' +# SAS specific +/read:/{print "rrd="$4"\nr_cor="$5"\nr_proc="$7"\nr_ucor="$8} +/write:/{print "rwr="$4"\nw_cor="$5"\nw_proc="$7"\nw_ucor="$8} +/Non-medium error count/{print "nonmed="$4} +/Elements in grown defect list/{print "defect="$6} + +# SAS common +/SAS/{type="sas"} +/Drive Temperature:/{print "temp="$4} +# Status can be a long string, substitute spaces for '_' +/SMART Health Status:/{printf "health="; for(i=4;i<=NF-1;i++){printf "%s_", $i}; printf "%s\n", $i} +/number of hours powered up/{print "hours_on="$7; hours_on=int($7)} +/Serial number:/{print "serial="$3} + +# SATA specific +/Reallocated_Sector_Ct/{print "realloc="$10} +/Reported_Uncorrect/{print "rep_ucor="$10} +/Command_Timeout/{print "cmd_to="$10} +/Current_Pending_Sector/{print "pend_sec="$10} +/Offline_Uncorrectable/{print "off_ucor="$10} +/ATA Error Count:/{print "ata_err="$4} +/Power_Cycle_Count/{print "pwr_cyc="$10} + +# SATA common +/SATA/{type="sata"} +/Temperature_Celsius/{print "temp="$10} +/Airflow_Temperature_Cel/{print "temp="$10} +/Current Temperature:/{print "temp="$3} +/SMART overall-health self-assessment test result:/{print "health="$6} +/Power_On_Hours/{print "hours_on="$10; hours_on=int($10)} +/Serial Number:/{print "serial="$3} + +# NVMe common +/NVMe/{type="nvme"} +/Temperature:/{print "temp="$2} +/SMART overall-health self-assessment test result:/{print "health="$6} +/Power On Hours:/{gsub("[^0-9]","",$4); print "hours_on="$4} +/Serial Number:/{print "serial="$3} +/Power Cycles:/{print "pwr_cyc="$3} + +# NVMe specific +/Media and Data Integrity Errors:/{print "nvme_err="$6} + +# SMART self-test info +/Self-test execution status:/{progress=tolower($4)} # SAS +/SMART Self-test log/{test_seen=1} # SAS +/SMART Extended Self-test Log/{test_seen=1} # SATA +/# 1/{ + test_type=tolower($3"_"$4); + # Status could be one word ("Completed") or multiple ("Completed: read + # failure"). Look for the ":" to see if we need to grab more words. + + if ($5 ~ ":") + status=tolower($5""$6"_"$7) + else + status=tolower($5) + if (status=="self") + status="running"; + + if (type == "sas") { + hours=int($(NF-4)) + } else { + hours=int($(NF-1)) + # SATA reports percent remaining, rather than percent done + # Convert it to percent done. + progress=(100-int($(NF-2)))"%" + } + # When we int()-ify "hours", it converts stuff like "NOW" and "-" into + # 0. In those cases, set it to hours_on, so they will cancel out in + # the "hours_ago" calculation later on. + if (hours == 0) + hours=hours_on + + if (test_seen) { + print "test="hours_on + print "test_type="test_type + print "test_status="status + print "test_progress="progress + } + # Not all drives report hours_on + if (hours_on && hours) { + total_hours_ago=(hours_on-hours) + days_ago=int(total_hours_ago/24) + hours_ago=(total_hours_ago % 24) + if (days_ago != 0) + ago_str=days_ago"d" + if (hours_ago !=0) + ago_str=ago_str""hours_ago"h" + print "test_ended="ago_str + } +} + +END {print "type="type; ORS="\n"; print ""} +'); +fi +type=$(echo "$out" | grep '^type=' | cut -d '=' -f 2) + +# If type is not set by now, either we don't have a block device +# or smartctl failed. Either way, default to ATA and set $out to +# nothing. +if [ -z "$type" ]; then + type="sata" + out= +fi + +case $script in +smart) + # Print temperature plus common predictors of drive failure + if [ "$type" = "sas" ] ; then + scripts="temp|health|r_ucor|w_ucor" + elif [ "$type" = "sata" ] ; then + scripts="temp|health|ata_err|realloc|rep_ucor|cmd_to|pend_sec|off_ucor" + elif [ "$type" = "nvme" ] ; then + scripts="temp|health|nvme_err" + fi + ;; +smartx) + # Print some other interesting stats + if [ "$type" = "sas" ] ; then + scripts="hours_on|defect|nonmed|r_proc|w_proc" + elif [ "$type" = "sata" ] ; then + scripts="hours_on|pwr_cyc" + elif [ "$type" = "nvme" ] ; then + scripts="hours_on|pwr_cyc" + fi + ;; +smart_test) + scripts="test_type|test_status|test_progress|test_ended" + ;; +*) + scripts="$script" +esac + +with_vals=$(echo "$out" | grep -E "$scripts") +if [ -n "$with_vals" ]; then + echo "$with_vals" + without_vals=$(echo "$scripts" | tr '|' '\n' | + grep -v -E "$(echo "$with_vals" | + awk -F "=" '{print $1}')" | awk '{print $0"="}') +else + without_vals=$(echo "$scripts" | tr '|' '\n' | awk '{print $0"="}') +fi + +if [ -n "$without_vals" ]; then + echo "$without_vals" +fi diff --git a/straper/db/public/zfs/etc-zfs/zpool.d/cmd_to b/straper/db/public/zfs/etc-zfs/zpool.d/cmd_to new file mode 100755 index 0000000..8ad3e10 --- /dev/null +++ b/straper/db/public/zfs/etc-zfs/zpool.d/cmd_to @@ -0,0 +1,249 @@ +#!/bin/sh +# +# Show SMART stats +# + +helpstr=" +smart: Show SMART temperature and error stats (specific to drive type) +smartx: Show SMART extended drive stats (specific to drive type). +temp: Show SMART drive temperature in celsius (all drives). +health: Show reported SMART status (all drives). +r_proc: Show SMART read GBytes processed over drive lifetime (SAS). +w_proc: Show SMART write GBytes processed over drive lifetime (SAS). +r_ucor: Show SMART read uncorrectable errors (SAS). +w_ucor: Show SMART write uncorrectable errors (SAS). +nonmed: Show SMART non-medium errors (SAS). +defect: Show SMART grown defect list (SAS). +hours_on: Show number of hours drive powered on (all drives). +realloc: Show SMART reallocated sectors count (ATA). +rep_ucor: Show SMART reported uncorrectable count (ATA). +cmd_to: Show SMART command timeout count (ATA). +pend_sec: Show SMART current pending sector count (ATA). +off_ucor: Show SMART offline uncorrectable errors (ATA). +ata_err: Show SMART ATA errors (ATA). +pwr_cyc: Show SMART power cycle count (ATA). +serial: Show disk serial number. +nvme_err: Show SMART NVMe errors (NVMe). +smart_test: Show SMART self-test results summary. +test_type: Show SMART self-test type (short, long... ). +test_status: Show SMART self-test status. +test_progress: Show SMART self-test percentage done. +test_ended: Show when the last SMART self-test ended (if supported). +" + +# Hack for developer testing +# +# If you set $samples to a directory containing smartctl output text files, +# we will use them instead of running smartctl on the vdevs. This can be +# useful if you want to test a bunch of different smartctl outputs. Also, if +# $samples is set, and additional 'file' column is added to the zpool output +# showing the filename. +samples= + +# get_filename_from_dir DIR +# +# Look in directory DIR and return a filename from it. The filename returned +# is chosen quasi-sequentially (based off our PID). This allows us to return +# a different filename every time this script is invoked (which we do for each +# vdev), without having to maintain state. +get_filename_from_dir() +{ + dir=$1 + pid="$$" + num_files=$(find "$dir" -maxdepth 1 -type f | wc -l) + mod=$((pid % num_files)) + i=0 + find "$dir" -type f -printf '%f\n' | while read -r file ; do + if [ "$mod" = "$i" ] ; then + echo "$file" + break + fi + i=$((i+1)) + done +} + +script="${0##*/}" + +if [ "$1" = "-h" ] ; then + echo "$helpstr" | grep "$script:" | tr -s '\t' | cut -f 2- + exit +fi + +# Sometimes, UPATH ends up /dev/(null). +# That should be corrected, but for now... +# shellcheck disable=SC2154 +if [ ! -b "$VDEV_UPATH" ]; then + somepath="${VDEV_PATH}" +else + somepath="${VDEV_UPATH}" +fi + +if [ -b "$somepath" ] && PATH="/usr/sbin:$PATH" command -v smartctl > /dev/null || [ -n "$samples" ] ; then + if [ -n "$samples" ] ; then + # cat a smartctl output text file instead of running smartctl + # on a vdev (only used for developer testing). + file=$(get_filename_from_dir "$samples") + echo "file=$file" + raw_out=$(cat "$samples/$file") + else + raw_out=$(sudo smartctl -a "$somepath") + fi + + # What kind of drive are we? Look for the right line in smartctl: + # + # SAS: + # Transport protocol: SAS + # + # SATA: + # ATA Version is: 8 + # + # NVMe: + # SMART/Health Information (NVMe Log 0xnn, NSID 0xnn) + # + out=$(echo "$raw_out" | awk ' +# SAS specific +/read:/{print "rrd="$4"\nr_cor="$5"\nr_proc="$7"\nr_ucor="$8} +/write:/{print "rwr="$4"\nw_cor="$5"\nw_proc="$7"\nw_ucor="$8} +/Non-medium error count/{print "nonmed="$4} +/Elements in grown defect list/{print "defect="$6} + +# SAS common +/SAS/{type="sas"} +/Drive Temperature:/{print "temp="$4} +# Status can be a long string, substitute spaces for '_' +/SMART Health Status:/{printf "health="; for(i=4;i<=NF-1;i++){printf "%s_", $i}; printf "%s\n", $i} +/number of hours powered up/{print "hours_on="$7; hours_on=int($7)} +/Serial number:/{print "serial="$3} + +# SATA specific +/Reallocated_Sector_Ct/{print "realloc="$10} +/Reported_Uncorrect/{print "rep_ucor="$10} +/Command_Timeout/{print "cmd_to="$10} +/Current_Pending_Sector/{print "pend_sec="$10} +/Offline_Uncorrectable/{print "off_ucor="$10} +/ATA Error Count:/{print "ata_err="$4} +/Power_Cycle_Count/{print "pwr_cyc="$10} + +# SATA common +/SATA/{type="sata"} +/Temperature_Celsius/{print "temp="$10} +/Airflow_Temperature_Cel/{print "temp="$10} +/Current Temperature:/{print "temp="$3} +/SMART overall-health self-assessment test result:/{print "health="$6} +/Power_On_Hours/{print "hours_on="$10; hours_on=int($10)} +/Serial Number:/{print "serial="$3} + +# NVMe common +/NVMe/{type="nvme"} +/Temperature:/{print "temp="$2} +/SMART overall-health self-assessment test result:/{print "health="$6} +/Power On Hours:/{gsub("[^0-9]","",$4); print "hours_on="$4} +/Serial Number:/{print "serial="$3} +/Power Cycles:/{print "pwr_cyc="$3} + +# NVMe specific +/Media and Data Integrity Errors:/{print "nvme_err="$6} + +# SMART self-test info +/Self-test execution status:/{progress=tolower($4)} # SAS +/SMART Self-test log/{test_seen=1} # SAS +/SMART Extended Self-test Log/{test_seen=1} # SATA +/# 1/{ + test_type=tolower($3"_"$4); + # Status could be one word ("Completed") or multiple ("Completed: read + # failure"). Look for the ":" to see if we need to grab more words. + + if ($5 ~ ":") + status=tolower($5""$6"_"$7) + else + status=tolower($5) + if (status=="self") + status="running"; + + if (type == "sas") { + hours=int($(NF-4)) + } else { + hours=int($(NF-1)) + # SATA reports percent remaining, rather than percent done + # Convert it to percent done. + progress=(100-int($(NF-2)))"%" + } + # When we int()-ify "hours", it converts stuff like "NOW" and "-" into + # 0. In those cases, set it to hours_on, so they will cancel out in + # the "hours_ago" calculation later on. + if (hours == 0) + hours=hours_on + + if (test_seen) { + print "test="hours_on + print "test_type="test_type + print "test_status="status + print "test_progress="progress + } + # Not all drives report hours_on + if (hours_on && hours) { + total_hours_ago=(hours_on-hours) + days_ago=int(total_hours_ago/24) + hours_ago=(total_hours_ago % 24) + if (days_ago != 0) + ago_str=days_ago"d" + if (hours_ago !=0) + ago_str=ago_str""hours_ago"h" + print "test_ended="ago_str + } +} + +END {print "type="type; ORS="\n"; print ""} +'); +fi +type=$(echo "$out" | grep '^type=' | cut -d '=' -f 2) + +# If type is not set by now, either we don't have a block device +# or smartctl failed. Either way, default to ATA and set $out to +# nothing. +if [ -z "$type" ]; then + type="sata" + out= +fi + +case $script in +smart) + # Print temperature plus common predictors of drive failure + if [ "$type" = "sas" ] ; then + scripts="temp|health|r_ucor|w_ucor" + elif [ "$type" = "sata" ] ; then + scripts="temp|health|ata_err|realloc|rep_ucor|cmd_to|pend_sec|off_ucor" + elif [ "$type" = "nvme" ] ; then + scripts="temp|health|nvme_err" + fi + ;; +smartx) + # Print some other interesting stats + if [ "$type" = "sas" ] ; then + scripts="hours_on|defect|nonmed|r_proc|w_proc" + elif [ "$type" = "sata" ] ; then + scripts="hours_on|pwr_cyc" + elif [ "$type" = "nvme" ] ; then + scripts="hours_on|pwr_cyc" + fi + ;; +smart_test) + scripts="test_type|test_status|test_progress|test_ended" + ;; +*) + scripts="$script" +esac + +with_vals=$(echo "$out" | grep -E "$scripts") +if [ -n "$with_vals" ]; then + echo "$with_vals" + without_vals=$(echo "$scripts" | tr '|' '\n' | + grep -v -E "$(echo "$with_vals" | + awk -F "=" '{print $1}')" | awk '{print $0"="}') +else + without_vals=$(echo "$scripts" | tr '|' '\n' | awk '{print $0"="}') +fi + +if [ -n "$without_vals" ]; then + echo "$without_vals" +fi diff --git a/straper/db/public/zfs/etc-zfs/zpool.d/defect b/straper/db/public/zfs/etc-zfs/zpool.d/defect new file mode 100755 index 0000000..8ad3e10 --- /dev/null +++ b/straper/db/public/zfs/etc-zfs/zpool.d/defect @@ -0,0 +1,249 @@ +#!/bin/sh +# +# Show SMART stats +# + +helpstr=" +smart: Show SMART temperature and error stats (specific to drive type) +smartx: Show SMART extended drive stats (specific to drive type). +temp: Show SMART drive temperature in celsius (all drives). +health: Show reported SMART status (all drives). +r_proc: Show SMART read GBytes processed over drive lifetime (SAS). +w_proc: Show SMART write GBytes processed over drive lifetime (SAS). +r_ucor: Show SMART read uncorrectable errors (SAS). +w_ucor: Show SMART write uncorrectable errors (SAS). +nonmed: Show SMART non-medium errors (SAS). +defect: Show SMART grown defect list (SAS). +hours_on: Show number of hours drive powered on (all drives). +realloc: Show SMART reallocated sectors count (ATA). +rep_ucor: Show SMART reported uncorrectable count (ATA). +cmd_to: Show SMART command timeout count (ATA). +pend_sec: Show SMART current pending sector count (ATA). +off_ucor: Show SMART offline uncorrectable errors (ATA). +ata_err: Show SMART ATA errors (ATA). +pwr_cyc: Show SMART power cycle count (ATA). +serial: Show disk serial number. +nvme_err: Show SMART NVMe errors (NVMe). +smart_test: Show SMART self-test results summary. +test_type: Show SMART self-test type (short, long... ). +test_status: Show SMART self-test status. +test_progress: Show SMART self-test percentage done. +test_ended: Show when the last SMART self-test ended (if supported). +" + +# Hack for developer testing +# +# If you set $samples to a directory containing smartctl output text files, +# we will use them instead of running smartctl on the vdevs. This can be +# useful if you want to test a bunch of different smartctl outputs. Also, if +# $samples is set, and additional 'file' column is added to the zpool output +# showing the filename. +samples= + +# get_filename_from_dir DIR +# +# Look in directory DIR and return a filename from it. The filename returned +# is chosen quasi-sequentially (based off our PID). This allows us to return +# a different filename every time this script is invoked (which we do for each +# vdev), without having to maintain state. +get_filename_from_dir() +{ + dir=$1 + pid="$$" + num_files=$(find "$dir" -maxdepth 1 -type f | wc -l) + mod=$((pid % num_files)) + i=0 + find "$dir" -type f -printf '%f\n' | while read -r file ; do + if [ "$mod" = "$i" ] ; then + echo "$file" + break + fi + i=$((i+1)) + done +} + +script="${0##*/}" + +if [ "$1" = "-h" ] ; then + echo "$helpstr" | grep "$script:" | tr -s '\t' | cut -f 2- + exit +fi + +# Sometimes, UPATH ends up /dev/(null). +# That should be corrected, but for now... +# shellcheck disable=SC2154 +if [ ! -b "$VDEV_UPATH" ]; then + somepath="${VDEV_PATH}" +else + somepath="${VDEV_UPATH}" +fi + +if [ -b "$somepath" ] && PATH="/usr/sbin:$PATH" command -v smartctl > /dev/null || [ -n "$samples" ] ; then + if [ -n "$samples" ] ; then + # cat a smartctl output text file instead of running smartctl + # on a vdev (only used for developer testing). + file=$(get_filename_from_dir "$samples") + echo "file=$file" + raw_out=$(cat "$samples/$file") + else + raw_out=$(sudo smartctl -a "$somepath") + fi + + # What kind of drive are we? Look for the right line in smartctl: + # + # SAS: + # Transport protocol: SAS + # + # SATA: + # ATA Version is: 8 + # + # NVMe: + # SMART/Health Information (NVMe Log 0xnn, NSID 0xnn) + # + out=$(echo "$raw_out" | awk ' +# SAS specific +/read:/{print "rrd="$4"\nr_cor="$5"\nr_proc="$7"\nr_ucor="$8} +/write:/{print "rwr="$4"\nw_cor="$5"\nw_proc="$7"\nw_ucor="$8} +/Non-medium error count/{print "nonmed="$4} +/Elements in grown defect list/{print "defect="$6} + +# SAS common +/SAS/{type="sas"} +/Drive Temperature:/{print "temp="$4} +# Status can be a long string, substitute spaces for '_' +/SMART Health Status:/{printf "health="; for(i=4;i<=NF-1;i++){printf "%s_", $i}; printf "%s\n", $i} +/number of hours powered up/{print "hours_on="$7; hours_on=int($7)} +/Serial number:/{print "serial="$3} + +# SATA specific +/Reallocated_Sector_Ct/{print "realloc="$10} +/Reported_Uncorrect/{print "rep_ucor="$10} +/Command_Timeout/{print "cmd_to="$10} +/Current_Pending_Sector/{print "pend_sec="$10} +/Offline_Uncorrectable/{print "off_ucor="$10} +/ATA Error Count:/{print "ata_err="$4} +/Power_Cycle_Count/{print "pwr_cyc="$10} + +# SATA common +/SATA/{type="sata"} +/Temperature_Celsius/{print "temp="$10} +/Airflow_Temperature_Cel/{print "temp="$10} +/Current Temperature:/{print "temp="$3} +/SMART overall-health self-assessment test result:/{print "health="$6} +/Power_On_Hours/{print "hours_on="$10; hours_on=int($10)} +/Serial Number:/{print "serial="$3} + +# NVMe common +/NVMe/{type="nvme"} +/Temperature:/{print "temp="$2} +/SMART overall-health self-assessment test result:/{print "health="$6} +/Power On Hours:/{gsub("[^0-9]","",$4); print "hours_on="$4} +/Serial Number:/{print "serial="$3} +/Power Cycles:/{print "pwr_cyc="$3} + +# NVMe specific +/Media and Data Integrity Errors:/{print "nvme_err="$6} + +# SMART self-test info +/Self-test execution status:/{progress=tolower($4)} # SAS +/SMART Self-test log/{test_seen=1} # SAS +/SMART Extended Self-test Log/{test_seen=1} # SATA +/# 1/{ + test_type=tolower($3"_"$4); + # Status could be one word ("Completed") or multiple ("Completed: read + # failure"). Look for the ":" to see if we need to grab more words. + + if ($5 ~ ":") + status=tolower($5""$6"_"$7) + else + status=tolower($5) + if (status=="self") + status="running"; + + if (type == "sas") { + hours=int($(NF-4)) + } else { + hours=int($(NF-1)) + # SATA reports percent remaining, rather than percent done + # Convert it to percent done. + progress=(100-int($(NF-2)))"%" + } + # When we int()-ify "hours", it converts stuff like "NOW" and "-" into + # 0. In those cases, set it to hours_on, so they will cancel out in + # the "hours_ago" calculation later on. + if (hours == 0) + hours=hours_on + + if (test_seen) { + print "test="hours_on + print "test_type="test_type + print "test_status="status + print "test_progress="progress + } + # Not all drives report hours_on + if (hours_on && hours) { + total_hours_ago=(hours_on-hours) + days_ago=int(total_hours_ago/24) + hours_ago=(total_hours_ago % 24) + if (days_ago != 0) + ago_str=days_ago"d" + if (hours_ago !=0) + ago_str=ago_str""hours_ago"h" + print "test_ended="ago_str + } +} + +END {print "type="type; ORS="\n"; print ""} +'); +fi +type=$(echo "$out" | grep '^type=' | cut -d '=' -f 2) + +# If type is not set by now, either we don't have a block device +# or smartctl failed. Either way, default to ATA and set $out to +# nothing. +if [ -z "$type" ]; then + type="sata" + out= +fi + +case $script in +smart) + # Print temperature plus common predictors of drive failure + if [ "$type" = "sas" ] ; then + scripts="temp|health|r_ucor|w_ucor" + elif [ "$type" = "sata" ] ; then + scripts="temp|health|ata_err|realloc|rep_ucor|cmd_to|pend_sec|off_ucor" + elif [ "$type" = "nvme" ] ; then + scripts="temp|health|nvme_err" + fi + ;; +smartx) + # Print some other interesting stats + if [ "$type" = "sas" ] ; then + scripts="hours_on|defect|nonmed|r_proc|w_proc" + elif [ "$type" = "sata" ] ; then + scripts="hours_on|pwr_cyc" + elif [ "$type" = "nvme" ] ; then + scripts="hours_on|pwr_cyc" + fi + ;; +smart_test) + scripts="test_type|test_status|test_progress|test_ended" + ;; +*) + scripts="$script" +esac + +with_vals=$(echo "$out" | grep -E "$scripts") +if [ -n "$with_vals" ]; then + echo "$with_vals" + without_vals=$(echo "$scripts" | tr '|' '\n' | + grep -v -E "$(echo "$with_vals" | + awk -F "=" '{print $1}')" | awk '{print $0"="}') +else + without_vals=$(echo "$scripts" | tr '|' '\n' | awk '{print $0"="}') +fi + +if [ -n "$without_vals" ]; then + echo "$without_vals" +fi diff --git a/straper/db/public/zfs/etc-zfs/zpool.d/dm-deps b/straper/db/public/zfs/etc-zfs/zpool.d/dm-deps new file mode 100755 index 0000000..4422478 --- /dev/null +++ b/straper/db/public/zfs/etc-zfs/zpool.d/dm-deps @@ -0,0 +1,28 @@ +#!/bin/sh +# +# Show device mapper dependent / underlying devices. This is useful for +# looking up the /dev/sd* devices associated with a dm or multipath device. +# + +if [ "$1" = "-h" ] ; then + echo "Show device mapper dependent (underlying) devices." + exit +fi + +# shellcheck disable=SC2154 +dev="$VDEV_PATH" + +# If the VDEV path is a symlink, resolve it to a real device +if [ -L "$dev" ] ; then + dev=$(readlink "$dev") +fi + +dev="${dev##*/}" +val="" +if [ -d "/sys/class/block/$dev/slaves" ] ; then + # ls -C: output in columns, no newlines, two spaces (change to one) + # shellcheck disable=SC2012 + val=$(ls -C "/sys/class/block/$dev/slaves" | tr -s '[:space:]' ' ') +fi + +echo "dm-deps=$val" diff --git a/straper/db/public/zfs/etc-zfs/zpool.d/enc b/straper/db/public/zfs/etc-zfs/zpool.d/enc new file mode 100755 index 0000000..19ef92a --- /dev/null +++ b/straper/db/public/zfs/etc-zfs/zpool.d/enc @@ -0,0 +1,69 @@ +#!/bin/sh +# +# Print SCSI Enclosure Services (SES) info. The output is dependent on the name +# of the script/symlink used to call it. +# +helpstr=" +enc: Show disk enclosure w:x:y:z value. +slot: Show disk slot number as reported by the enclosure. +encdev: Show /dev/sg* device associated with the enclosure disk slot. +fault_led: Show value of the disk enclosure slot fault LED. +locate_led: Show value of the disk enclosure slot locate LED. +ses: Show disk's enc, enc device, slot, and fault/locate LED values." + +script="${0##*/}" +if [ "$1" = "-h" ] ; then + echo "$helpstr" | grep "$script:" | tr -s '\t' | cut -f 2- + exit +fi + +if [ "$script" = "ses" ] ; then + scripts='enc encdev slot fault_led locate_led' +else + scripts="$script" +fi + +for i in $scripts ; do + # shellcheck disable=SC2154 + if [ -z "$VDEV_ENC_SYSFS_PATH" ] ; then + echo "$i=" + continue + fi + + val="" + case $i in + enc) + if echo "$VDEV_ENC_SYSFS_PATH" | grep -q '/sys/bus/pci/slots' ; then + val="$VDEV_ENC_SYSFS_PATH" + else + val="$(ls """$VDEV_ENC_SYSFS_PATH/../../""" 2>/dev/null)" + fi + ;; + slot) + if echo "$VDEV_ENC_SYSFS_PATH" | grep -q '/sys/bus/pci/slots' ; then + val="$(basename """$VDEV_ENC_SYSFS_PATH""")" + else + val="$(cat """$VDEV_ENC_SYSFS_PATH/slot""" 2>/dev/null)" + fi + ;; + encdev) + val=$(ls "$VDEV_ENC_SYSFS_PATH/../device/scsi_generic" 2>/dev/null) + ;; + fault_led) + # JBODs fault LED is called 'fault', NVMe fault LED is called + # 'attention'. + if [ -f "$VDEV_ENC_SYSFS_PATH/fault" ] ; then + val=$(cat "$VDEV_ENC_SYSFS_PATH/fault" 2>/dev/null) + elif [ -f "$VDEV_ENC_SYSFS_PATH/attention" ] ; then + val=$(cat "$VDEV_ENC_SYSFS_PATH/attention" 2>/dev/null) + fi + ;; + locate_led) + val=$(cat "$VDEV_ENC_SYSFS_PATH/locate" 2>/dev/null) + ;; + *) + val=invalid + ;; + esac + echo "$i=$val" +done diff --git a/straper/db/public/zfs/etc-zfs/zpool.d/encdev b/straper/db/public/zfs/etc-zfs/zpool.d/encdev new file mode 100755 index 0000000..19ef92a --- /dev/null +++ b/straper/db/public/zfs/etc-zfs/zpool.d/encdev @@ -0,0 +1,69 @@ +#!/bin/sh +# +# Print SCSI Enclosure Services (SES) info. The output is dependent on the name +# of the script/symlink used to call it. +# +helpstr=" +enc: Show disk enclosure w:x:y:z value. +slot: Show disk slot number as reported by the enclosure. +encdev: Show /dev/sg* device associated with the enclosure disk slot. +fault_led: Show value of the disk enclosure slot fault LED. +locate_led: Show value of the disk enclosure slot locate LED. +ses: Show disk's enc, enc device, slot, and fault/locate LED values." + +script="${0##*/}" +if [ "$1" = "-h" ] ; then + echo "$helpstr" | grep "$script:" | tr -s '\t' | cut -f 2- + exit +fi + +if [ "$script" = "ses" ] ; then + scripts='enc encdev slot fault_led locate_led' +else + scripts="$script" +fi + +for i in $scripts ; do + # shellcheck disable=SC2154 + if [ -z "$VDEV_ENC_SYSFS_PATH" ] ; then + echo "$i=" + continue + fi + + val="" + case $i in + enc) + if echo "$VDEV_ENC_SYSFS_PATH" | grep -q '/sys/bus/pci/slots' ; then + val="$VDEV_ENC_SYSFS_PATH" + else + val="$(ls """$VDEV_ENC_SYSFS_PATH/../../""" 2>/dev/null)" + fi + ;; + slot) + if echo "$VDEV_ENC_SYSFS_PATH" | grep -q '/sys/bus/pci/slots' ; then + val="$(basename """$VDEV_ENC_SYSFS_PATH""")" + else + val="$(cat """$VDEV_ENC_SYSFS_PATH/slot""" 2>/dev/null)" + fi + ;; + encdev) + val=$(ls "$VDEV_ENC_SYSFS_PATH/../device/scsi_generic" 2>/dev/null) + ;; + fault_led) + # JBODs fault LED is called 'fault', NVMe fault LED is called + # 'attention'. + if [ -f "$VDEV_ENC_SYSFS_PATH/fault" ] ; then + val=$(cat "$VDEV_ENC_SYSFS_PATH/fault" 2>/dev/null) + elif [ -f "$VDEV_ENC_SYSFS_PATH/attention" ] ; then + val=$(cat "$VDEV_ENC_SYSFS_PATH/attention" 2>/dev/null) + fi + ;; + locate_led) + val=$(cat "$VDEV_ENC_SYSFS_PATH/locate" 2>/dev/null) + ;; + *) + val=invalid + ;; + esac + echo "$i=$val" +done diff --git a/straper/db/public/zfs/etc-zfs/zpool.d/fault_led b/straper/db/public/zfs/etc-zfs/zpool.d/fault_led new file mode 100755 index 0000000..19ef92a --- /dev/null +++ b/straper/db/public/zfs/etc-zfs/zpool.d/fault_led @@ -0,0 +1,69 @@ +#!/bin/sh +# +# Print SCSI Enclosure Services (SES) info. The output is dependent on the name +# of the script/symlink used to call it. +# +helpstr=" +enc: Show disk enclosure w:x:y:z value. +slot: Show disk slot number as reported by the enclosure. +encdev: Show /dev/sg* device associated with the enclosure disk slot. +fault_led: Show value of the disk enclosure slot fault LED. +locate_led: Show value of the disk enclosure slot locate LED. +ses: Show disk's enc, enc device, slot, and fault/locate LED values." + +script="${0##*/}" +if [ "$1" = "-h" ] ; then + echo "$helpstr" | grep "$script:" | tr -s '\t' | cut -f 2- + exit +fi + +if [ "$script" = "ses" ] ; then + scripts='enc encdev slot fault_led locate_led' +else + scripts="$script" +fi + +for i in $scripts ; do + # shellcheck disable=SC2154 + if [ -z "$VDEV_ENC_SYSFS_PATH" ] ; then + echo "$i=" + continue + fi + + val="" + case $i in + enc) + if echo "$VDEV_ENC_SYSFS_PATH" | grep -q '/sys/bus/pci/slots' ; then + val="$VDEV_ENC_SYSFS_PATH" + else + val="$(ls """$VDEV_ENC_SYSFS_PATH/../../""" 2>/dev/null)" + fi + ;; + slot) + if echo "$VDEV_ENC_SYSFS_PATH" | grep -q '/sys/bus/pci/slots' ; then + val="$(basename """$VDEV_ENC_SYSFS_PATH""")" + else + val="$(cat """$VDEV_ENC_SYSFS_PATH/slot""" 2>/dev/null)" + fi + ;; + encdev) + val=$(ls "$VDEV_ENC_SYSFS_PATH/../device/scsi_generic" 2>/dev/null) + ;; + fault_led) + # JBODs fault LED is called 'fault', NVMe fault LED is called + # 'attention'. + if [ -f "$VDEV_ENC_SYSFS_PATH/fault" ] ; then + val=$(cat "$VDEV_ENC_SYSFS_PATH/fault" 2>/dev/null) + elif [ -f "$VDEV_ENC_SYSFS_PATH/attention" ] ; then + val=$(cat "$VDEV_ENC_SYSFS_PATH/attention" 2>/dev/null) + fi + ;; + locate_led) + val=$(cat "$VDEV_ENC_SYSFS_PATH/locate" 2>/dev/null) + ;; + *) + val=invalid + ;; + esac + echo "$i=$val" +done diff --git a/straper/db/public/zfs/etc-zfs/zpool.d/health b/straper/db/public/zfs/etc-zfs/zpool.d/health new file mode 100755 index 0000000..8ad3e10 --- /dev/null +++ b/straper/db/public/zfs/etc-zfs/zpool.d/health @@ -0,0 +1,249 @@ +#!/bin/sh +# +# Show SMART stats +# + +helpstr=" +smart: Show SMART temperature and error stats (specific to drive type) +smartx: Show SMART extended drive stats (specific to drive type). +temp: Show SMART drive temperature in celsius (all drives). +health: Show reported SMART status (all drives). +r_proc: Show SMART read GBytes processed over drive lifetime (SAS). +w_proc: Show SMART write GBytes processed over drive lifetime (SAS). +r_ucor: Show SMART read uncorrectable errors (SAS). +w_ucor: Show SMART write uncorrectable errors (SAS). +nonmed: Show SMART non-medium errors (SAS). +defect: Show SMART grown defect list (SAS). +hours_on: Show number of hours drive powered on (all drives). +realloc: Show SMART reallocated sectors count (ATA). +rep_ucor: Show SMART reported uncorrectable count (ATA). +cmd_to: Show SMART command timeout count (ATA). +pend_sec: Show SMART current pending sector count (ATA). +off_ucor: Show SMART offline uncorrectable errors (ATA). +ata_err: Show SMART ATA errors (ATA). +pwr_cyc: Show SMART power cycle count (ATA). +serial: Show disk serial number. +nvme_err: Show SMART NVMe errors (NVMe). +smart_test: Show SMART self-test results summary. +test_type: Show SMART self-test type (short, long... ). +test_status: Show SMART self-test status. +test_progress: Show SMART self-test percentage done. +test_ended: Show when the last SMART self-test ended (if supported). +" + +# Hack for developer testing +# +# If you set $samples to a directory containing smartctl output text files, +# we will use them instead of running smartctl on the vdevs. This can be +# useful if you want to test a bunch of different smartctl outputs. Also, if +# $samples is set, and additional 'file' column is added to the zpool output +# showing the filename. +samples= + +# get_filename_from_dir DIR +# +# Look in directory DIR and return a filename from it. The filename returned +# is chosen quasi-sequentially (based off our PID). This allows us to return +# a different filename every time this script is invoked (which we do for each +# vdev), without having to maintain state. +get_filename_from_dir() +{ + dir=$1 + pid="$$" + num_files=$(find "$dir" -maxdepth 1 -type f | wc -l) + mod=$((pid % num_files)) + i=0 + find "$dir" -type f -printf '%f\n' | while read -r file ; do + if [ "$mod" = "$i" ] ; then + echo "$file" + break + fi + i=$((i+1)) + done +} + +script="${0##*/}" + +if [ "$1" = "-h" ] ; then + echo "$helpstr" | grep "$script:" | tr -s '\t' | cut -f 2- + exit +fi + +# Sometimes, UPATH ends up /dev/(null). +# That should be corrected, but for now... +# shellcheck disable=SC2154 +if [ ! -b "$VDEV_UPATH" ]; then + somepath="${VDEV_PATH}" +else + somepath="${VDEV_UPATH}" +fi + +if [ -b "$somepath" ] && PATH="/usr/sbin:$PATH" command -v smartctl > /dev/null || [ -n "$samples" ] ; then + if [ -n "$samples" ] ; then + # cat a smartctl output text file instead of running smartctl + # on a vdev (only used for developer testing). + file=$(get_filename_from_dir "$samples") + echo "file=$file" + raw_out=$(cat "$samples/$file") + else + raw_out=$(sudo smartctl -a "$somepath") + fi + + # What kind of drive are we? Look for the right line in smartctl: + # + # SAS: + # Transport protocol: SAS + # + # SATA: + # ATA Version is: 8 + # + # NVMe: + # SMART/Health Information (NVMe Log 0xnn, NSID 0xnn) + # + out=$(echo "$raw_out" | awk ' +# SAS specific +/read:/{print "rrd="$4"\nr_cor="$5"\nr_proc="$7"\nr_ucor="$8} +/write:/{print "rwr="$4"\nw_cor="$5"\nw_proc="$7"\nw_ucor="$8} +/Non-medium error count/{print "nonmed="$4} +/Elements in grown defect list/{print "defect="$6} + +# SAS common +/SAS/{type="sas"} +/Drive Temperature:/{print "temp="$4} +# Status can be a long string, substitute spaces for '_' +/SMART Health Status:/{printf "health="; for(i=4;i<=NF-1;i++){printf "%s_", $i}; printf "%s\n", $i} +/number of hours powered up/{print "hours_on="$7; hours_on=int($7)} +/Serial number:/{print "serial="$3} + +# SATA specific +/Reallocated_Sector_Ct/{print "realloc="$10} +/Reported_Uncorrect/{print "rep_ucor="$10} +/Command_Timeout/{print "cmd_to="$10} +/Current_Pending_Sector/{print "pend_sec="$10} +/Offline_Uncorrectable/{print "off_ucor="$10} +/ATA Error Count:/{print "ata_err="$4} +/Power_Cycle_Count/{print "pwr_cyc="$10} + +# SATA common +/SATA/{type="sata"} +/Temperature_Celsius/{print "temp="$10} +/Airflow_Temperature_Cel/{print "temp="$10} +/Current Temperature:/{print "temp="$3} +/SMART overall-health self-assessment test result:/{print "health="$6} +/Power_On_Hours/{print "hours_on="$10; hours_on=int($10)} +/Serial Number:/{print "serial="$3} + +# NVMe common +/NVMe/{type="nvme"} +/Temperature:/{print "temp="$2} +/SMART overall-health self-assessment test result:/{print "health="$6} +/Power On Hours:/{gsub("[^0-9]","",$4); print "hours_on="$4} +/Serial Number:/{print "serial="$3} +/Power Cycles:/{print "pwr_cyc="$3} + +# NVMe specific +/Media and Data Integrity Errors:/{print "nvme_err="$6} + +# SMART self-test info +/Self-test execution status:/{progress=tolower($4)} # SAS +/SMART Self-test log/{test_seen=1} # SAS +/SMART Extended Self-test Log/{test_seen=1} # SATA +/# 1/{ + test_type=tolower($3"_"$4); + # Status could be one word ("Completed") or multiple ("Completed: read + # failure"). Look for the ":" to see if we need to grab more words. + + if ($5 ~ ":") + status=tolower($5""$6"_"$7) + else + status=tolower($5) + if (status=="self") + status="running"; + + if (type == "sas") { + hours=int($(NF-4)) + } else { + hours=int($(NF-1)) + # SATA reports percent remaining, rather than percent done + # Convert it to percent done. + progress=(100-int($(NF-2)))"%" + } + # When we int()-ify "hours", it converts stuff like "NOW" and "-" into + # 0. In those cases, set it to hours_on, so they will cancel out in + # the "hours_ago" calculation later on. + if (hours == 0) + hours=hours_on + + if (test_seen) { + print "test="hours_on + print "test_type="test_type + print "test_status="status + print "test_progress="progress + } + # Not all drives report hours_on + if (hours_on && hours) { + total_hours_ago=(hours_on-hours) + days_ago=int(total_hours_ago/24) + hours_ago=(total_hours_ago % 24) + if (days_ago != 0) + ago_str=days_ago"d" + if (hours_ago !=0) + ago_str=ago_str""hours_ago"h" + print "test_ended="ago_str + } +} + +END {print "type="type; ORS="\n"; print ""} +'); +fi +type=$(echo "$out" | grep '^type=' | cut -d '=' -f 2) + +# If type is not set by now, either we don't have a block device +# or smartctl failed. Either way, default to ATA and set $out to +# nothing. +if [ -z "$type" ]; then + type="sata" + out= +fi + +case $script in +smart) + # Print temperature plus common predictors of drive failure + if [ "$type" = "sas" ] ; then + scripts="temp|health|r_ucor|w_ucor" + elif [ "$type" = "sata" ] ; then + scripts="temp|health|ata_err|realloc|rep_ucor|cmd_to|pend_sec|off_ucor" + elif [ "$type" = "nvme" ] ; then + scripts="temp|health|nvme_err" + fi + ;; +smartx) + # Print some other interesting stats + if [ "$type" = "sas" ] ; then + scripts="hours_on|defect|nonmed|r_proc|w_proc" + elif [ "$type" = "sata" ] ; then + scripts="hours_on|pwr_cyc" + elif [ "$type" = "nvme" ] ; then + scripts="hours_on|pwr_cyc" + fi + ;; +smart_test) + scripts="test_type|test_status|test_progress|test_ended" + ;; +*) + scripts="$script" +esac + +with_vals=$(echo "$out" | grep -E "$scripts") +if [ -n "$with_vals" ]; then + echo "$with_vals" + without_vals=$(echo "$scripts" | tr '|' '\n' | + grep -v -E "$(echo "$with_vals" | + awk -F "=" '{print $1}')" | awk '{print $0"="}') +else + without_vals=$(echo "$scripts" | tr '|' '\n' | awk '{print $0"="}') +fi + +if [ -n "$without_vals" ]; then + echo "$without_vals" +fi diff --git a/straper/db/public/zfs/etc-zfs/zpool.d/hours_on b/straper/db/public/zfs/etc-zfs/zpool.d/hours_on new file mode 100755 index 0000000..8ad3e10 --- /dev/null +++ b/straper/db/public/zfs/etc-zfs/zpool.d/hours_on @@ -0,0 +1,249 @@ +#!/bin/sh +# +# Show SMART stats +# + +helpstr=" +smart: Show SMART temperature and error stats (specific to drive type) +smartx: Show SMART extended drive stats (specific to drive type). +temp: Show SMART drive temperature in celsius (all drives). +health: Show reported SMART status (all drives). +r_proc: Show SMART read GBytes processed over drive lifetime (SAS). +w_proc: Show SMART write GBytes processed over drive lifetime (SAS). +r_ucor: Show SMART read uncorrectable errors (SAS). +w_ucor: Show SMART write uncorrectable errors (SAS). +nonmed: Show SMART non-medium errors (SAS). +defect: Show SMART grown defect list (SAS). +hours_on: Show number of hours drive powered on (all drives). +realloc: Show SMART reallocated sectors count (ATA). +rep_ucor: Show SMART reported uncorrectable count (ATA). +cmd_to: Show SMART command timeout count (ATA). +pend_sec: Show SMART current pending sector count (ATA). +off_ucor: Show SMART offline uncorrectable errors (ATA). +ata_err: Show SMART ATA errors (ATA). +pwr_cyc: Show SMART power cycle count (ATA). +serial: Show disk serial number. +nvme_err: Show SMART NVMe errors (NVMe). +smart_test: Show SMART self-test results summary. +test_type: Show SMART self-test type (short, long... ). +test_status: Show SMART self-test status. +test_progress: Show SMART self-test percentage done. +test_ended: Show when the last SMART self-test ended (if supported). +" + +# Hack for developer testing +# +# If you set $samples to a directory containing smartctl output text files, +# we will use them instead of running smartctl on the vdevs. This can be +# useful if you want to test a bunch of different smartctl outputs. Also, if +# $samples is set, and additional 'file' column is added to the zpool output +# showing the filename. +samples= + +# get_filename_from_dir DIR +# +# Look in directory DIR and return a filename from it. The filename returned +# is chosen quasi-sequentially (based off our PID). This allows us to return +# a different filename every time this script is invoked (which we do for each +# vdev), without having to maintain state. +get_filename_from_dir() +{ + dir=$1 + pid="$$" + num_files=$(find "$dir" -maxdepth 1 -type f | wc -l) + mod=$((pid % num_files)) + i=0 + find "$dir" -type f -printf '%f\n' | while read -r file ; do + if [ "$mod" = "$i" ] ; then + echo "$file" + break + fi + i=$((i+1)) + done +} + +script="${0##*/}" + +if [ "$1" = "-h" ] ; then + echo "$helpstr" | grep "$script:" | tr -s '\t' | cut -f 2- + exit +fi + +# Sometimes, UPATH ends up /dev/(null). +# That should be corrected, but for now... +# shellcheck disable=SC2154 +if [ ! -b "$VDEV_UPATH" ]; then + somepath="${VDEV_PATH}" +else + somepath="${VDEV_UPATH}" +fi + +if [ -b "$somepath" ] && PATH="/usr/sbin:$PATH" command -v smartctl > /dev/null || [ -n "$samples" ] ; then + if [ -n "$samples" ] ; then + # cat a smartctl output text file instead of running smartctl + # on a vdev (only used for developer testing). + file=$(get_filename_from_dir "$samples") + echo "file=$file" + raw_out=$(cat "$samples/$file") + else + raw_out=$(sudo smartctl -a "$somepath") + fi + + # What kind of drive are we? Look for the right line in smartctl: + # + # SAS: + # Transport protocol: SAS + # + # SATA: + # ATA Version is: 8 + # + # NVMe: + # SMART/Health Information (NVMe Log 0xnn, NSID 0xnn) + # + out=$(echo "$raw_out" | awk ' +# SAS specific +/read:/{print "rrd="$4"\nr_cor="$5"\nr_proc="$7"\nr_ucor="$8} +/write:/{print "rwr="$4"\nw_cor="$5"\nw_proc="$7"\nw_ucor="$8} +/Non-medium error count/{print "nonmed="$4} +/Elements in grown defect list/{print "defect="$6} + +# SAS common +/SAS/{type="sas"} +/Drive Temperature:/{print "temp="$4} +# Status can be a long string, substitute spaces for '_' +/SMART Health Status:/{printf "health="; for(i=4;i<=NF-1;i++){printf "%s_", $i}; printf "%s\n", $i} +/number of hours powered up/{print "hours_on="$7; hours_on=int($7)} +/Serial number:/{print "serial="$3} + +# SATA specific +/Reallocated_Sector_Ct/{print "realloc="$10} +/Reported_Uncorrect/{print "rep_ucor="$10} +/Command_Timeout/{print "cmd_to="$10} +/Current_Pending_Sector/{print "pend_sec="$10} +/Offline_Uncorrectable/{print "off_ucor="$10} +/ATA Error Count:/{print "ata_err="$4} +/Power_Cycle_Count/{print "pwr_cyc="$10} + +# SATA common +/SATA/{type="sata"} +/Temperature_Celsius/{print "temp="$10} +/Airflow_Temperature_Cel/{print "temp="$10} +/Current Temperature:/{print "temp="$3} +/SMART overall-health self-assessment test result:/{print "health="$6} +/Power_On_Hours/{print "hours_on="$10; hours_on=int($10)} +/Serial Number:/{print "serial="$3} + +# NVMe common +/NVMe/{type="nvme"} +/Temperature:/{print "temp="$2} +/SMART overall-health self-assessment test result:/{print "health="$6} +/Power On Hours:/{gsub("[^0-9]","",$4); print "hours_on="$4} +/Serial Number:/{print "serial="$3} +/Power Cycles:/{print "pwr_cyc="$3} + +# NVMe specific +/Media and Data Integrity Errors:/{print "nvme_err="$6} + +# SMART self-test info +/Self-test execution status:/{progress=tolower($4)} # SAS +/SMART Self-test log/{test_seen=1} # SAS +/SMART Extended Self-test Log/{test_seen=1} # SATA +/# 1/{ + test_type=tolower($3"_"$4); + # Status could be one word ("Completed") or multiple ("Completed: read + # failure"). Look for the ":" to see if we need to grab more words. + + if ($5 ~ ":") + status=tolower($5""$6"_"$7) + else + status=tolower($5) + if (status=="self") + status="running"; + + if (type == "sas") { + hours=int($(NF-4)) + } else { + hours=int($(NF-1)) + # SATA reports percent remaining, rather than percent done + # Convert it to percent done. + progress=(100-int($(NF-2)))"%" + } + # When we int()-ify "hours", it converts stuff like "NOW" and "-" into + # 0. In those cases, set it to hours_on, so they will cancel out in + # the "hours_ago" calculation later on. + if (hours == 0) + hours=hours_on + + if (test_seen) { + print "test="hours_on + print "test_type="test_type + print "test_status="status + print "test_progress="progress + } + # Not all drives report hours_on + if (hours_on && hours) { + total_hours_ago=(hours_on-hours) + days_ago=int(total_hours_ago/24) + hours_ago=(total_hours_ago % 24) + if (days_ago != 0) + ago_str=days_ago"d" + if (hours_ago !=0) + ago_str=ago_str""hours_ago"h" + print "test_ended="ago_str + } +} + +END {print "type="type; ORS="\n"; print ""} +'); +fi +type=$(echo "$out" | grep '^type=' | cut -d '=' -f 2) + +# If type is not set by now, either we don't have a block device +# or smartctl failed. Either way, default to ATA and set $out to +# nothing. +if [ -z "$type" ]; then + type="sata" + out= +fi + +case $script in +smart) + # Print temperature plus common predictors of drive failure + if [ "$type" = "sas" ] ; then + scripts="temp|health|r_ucor|w_ucor" + elif [ "$type" = "sata" ] ; then + scripts="temp|health|ata_err|realloc|rep_ucor|cmd_to|pend_sec|off_ucor" + elif [ "$type" = "nvme" ] ; then + scripts="temp|health|nvme_err" + fi + ;; +smartx) + # Print some other interesting stats + if [ "$type" = "sas" ] ; then + scripts="hours_on|defect|nonmed|r_proc|w_proc" + elif [ "$type" = "sata" ] ; then + scripts="hours_on|pwr_cyc" + elif [ "$type" = "nvme" ] ; then + scripts="hours_on|pwr_cyc" + fi + ;; +smart_test) + scripts="test_type|test_status|test_progress|test_ended" + ;; +*) + scripts="$script" +esac + +with_vals=$(echo "$out" | grep -E "$scripts") +if [ -n "$with_vals" ]; then + echo "$with_vals" + without_vals=$(echo "$scripts" | tr '|' '\n' | + grep -v -E "$(echo "$with_vals" | + awk -F "=" '{print $1}')" | awk '{print $0"="}') +else + without_vals=$(echo "$scripts" | tr '|' '\n' | awk '{print $0"="}') +fi + +if [ -n "$without_vals" ]; then + echo "$without_vals" +fi diff --git a/straper/db/public/zfs/etc-zfs/zpool.d/iostat b/straper/db/public/zfs/etc-zfs/zpool.d/iostat new file mode 100755 index 0000000..2f8d79a --- /dev/null +++ b/straper/db/public/zfs/etc-zfs/zpool.d/iostat @@ -0,0 +1,86 @@ +#!/bin/sh +# +# Display most relevant iostat bandwidth/latency numbers. The output is +# dependent on the name of the script/symlink used to call it. +# + +helpstr=" +iostat: Show iostat values since boot (summary page). +iostat-1s: Do a single 1-second iostat sample and show values. +iostat-10s: Do a single 10-second iostat sample and show values." + +script="${0##*/}" +if [ "$1" = "-h" ] ; then + echo "$helpstr" | grep "$script:" | tr -s '\t' | cut -f 2- + exit +fi + +# Sometimes, UPATH ends up /dev/(null). +# That should be corrected, but for now... +# shellcheck disable=SC2154 +if [ ! -b "$VDEV_UPATH" ]; then + somepath="${VDEV_PATH}" +else + somepath="${VDEV_UPATH}" +fi + +if [ "$script" = "iostat-1s" ] ; then + # Do a single one-second sample + interval=1 + # Don't show summary stats + brief="yes" +elif [ "$script" = "iostat-10s" ] ; then + # Do a single ten-second sample + interval=10 + # Don't show summary stats + brief="yes" +fi + +if [ -f "$somepath" ] ; then + # We're a file-based vdev, iostat doesn't work on us. Do nothing. + exit +fi + +if [ "$(uname)" = "FreeBSD" ]; then + out=$(iostat -dKx \ + ${interval:+"-w $interval"} \ + ${interval:+"-c 1"} \ + "$somepath" | tail -n 2) +else + out=$(iostat -kx \ + ${brief:+"-y"} \ + ${interval:+"$interval"} \ + ${interval:+"1"} \ + "$somepath" | grep -v '^$' | tail -n 2) +fi + + +# Sample output (we want the last two lines): +# +# Linux 2.6.32-642.13.1.el6.x86_64 (centos68) 03/09/2017 _x86_64_ (6 CPU) +# +# avg-cpu: %user %nice %system %iowait %steal %idle +# 0.00 0.00 0.00 0.00 0.00 100.00 +# +# Device: rrqm/s wrqm/s r/s w/s rkB/s wkB/s avgrq-sz avgqu-sz await r_await w_await svctm %util +# sdb 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 +# + +# Get the column names +cols=$(echo "$out" | head -n 1) + +# Get the values and tab separate them to make them cut-able. +vals=$(echo "$out" | tail -n 1 | tr -s '[:space:]' '\t') + +i=0 +for col in $cols ; do + i=$((i+1)) + # Skip the first column since it's just the device name + if [ "$i" -eq 1 ]; then + continue + fi + + # Get i'th value + val=$(echo "$vals" | cut -f "$i") + echo "$col=$val" +done diff --git a/straper/db/public/zfs/etc-zfs/zpool.d/iostat-10s b/straper/db/public/zfs/etc-zfs/zpool.d/iostat-10s new file mode 100755 index 0000000..2f8d79a --- /dev/null +++ b/straper/db/public/zfs/etc-zfs/zpool.d/iostat-10s @@ -0,0 +1,86 @@ +#!/bin/sh +# +# Display most relevant iostat bandwidth/latency numbers. The output is +# dependent on the name of the script/symlink used to call it. +# + +helpstr=" +iostat: Show iostat values since boot (summary page). +iostat-1s: Do a single 1-second iostat sample and show values. +iostat-10s: Do a single 10-second iostat sample and show values." + +script="${0##*/}" +if [ "$1" = "-h" ] ; then + echo "$helpstr" | grep "$script:" | tr -s '\t' | cut -f 2- + exit +fi + +# Sometimes, UPATH ends up /dev/(null). +# That should be corrected, but for now... +# shellcheck disable=SC2154 +if [ ! -b "$VDEV_UPATH" ]; then + somepath="${VDEV_PATH}" +else + somepath="${VDEV_UPATH}" +fi + +if [ "$script" = "iostat-1s" ] ; then + # Do a single one-second sample + interval=1 + # Don't show summary stats + brief="yes" +elif [ "$script" = "iostat-10s" ] ; then + # Do a single ten-second sample + interval=10 + # Don't show summary stats + brief="yes" +fi + +if [ -f "$somepath" ] ; then + # We're a file-based vdev, iostat doesn't work on us. Do nothing. + exit +fi + +if [ "$(uname)" = "FreeBSD" ]; then + out=$(iostat -dKx \ + ${interval:+"-w $interval"} \ + ${interval:+"-c 1"} \ + "$somepath" | tail -n 2) +else + out=$(iostat -kx \ + ${brief:+"-y"} \ + ${interval:+"$interval"} \ + ${interval:+"1"} \ + "$somepath" | grep -v '^$' | tail -n 2) +fi + + +# Sample output (we want the last two lines): +# +# Linux 2.6.32-642.13.1.el6.x86_64 (centos68) 03/09/2017 _x86_64_ (6 CPU) +# +# avg-cpu: %user %nice %system %iowait %steal %idle +# 0.00 0.00 0.00 0.00 0.00 100.00 +# +# Device: rrqm/s wrqm/s r/s w/s rkB/s wkB/s avgrq-sz avgqu-sz await r_await w_await svctm %util +# sdb 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 +# + +# Get the column names +cols=$(echo "$out" | head -n 1) + +# Get the values and tab separate them to make them cut-able. +vals=$(echo "$out" | tail -n 1 | tr -s '[:space:]' '\t') + +i=0 +for col in $cols ; do + i=$((i+1)) + # Skip the first column since it's just the device name + if [ "$i" -eq 1 ]; then + continue + fi + + # Get i'th value + val=$(echo "$vals" | cut -f "$i") + echo "$col=$val" +done diff --git a/straper/db/public/zfs/etc-zfs/zpool.d/iostat-1s b/straper/db/public/zfs/etc-zfs/zpool.d/iostat-1s new file mode 100755 index 0000000..2f8d79a --- /dev/null +++ b/straper/db/public/zfs/etc-zfs/zpool.d/iostat-1s @@ -0,0 +1,86 @@ +#!/bin/sh +# +# Display most relevant iostat bandwidth/latency numbers. The output is +# dependent on the name of the script/symlink used to call it. +# + +helpstr=" +iostat: Show iostat values since boot (summary page). +iostat-1s: Do a single 1-second iostat sample and show values. +iostat-10s: Do a single 10-second iostat sample and show values." + +script="${0##*/}" +if [ "$1" = "-h" ] ; then + echo "$helpstr" | grep "$script:" | tr -s '\t' | cut -f 2- + exit +fi + +# Sometimes, UPATH ends up /dev/(null). +# That should be corrected, but for now... +# shellcheck disable=SC2154 +if [ ! -b "$VDEV_UPATH" ]; then + somepath="${VDEV_PATH}" +else + somepath="${VDEV_UPATH}" +fi + +if [ "$script" = "iostat-1s" ] ; then + # Do a single one-second sample + interval=1 + # Don't show summary stats + brief="yes" +elif [ "$script" = "iostat-10s" ] ; then + # Do a single ten-second sample + interval=10 + # Don't show summary stats + brief="yes" +fi + +if [ -f "$somepath" ] ; then + # We're a file-based vdev, iostat doesn't work on us. Do nothing. + exit +fi + +if [ "$(uname)" = "FreeBSD" ]; then + out=$(iostat -dKx \ + ${interval:+"-w $interval"} \ + ${interval:+"-c 1"} \ + "$somepath" | tail -n 2) +else + out=$(iostat -kx \ + ${brief:+"-y"} \ + ${interval:+"$interval"} \ + ${interval:+"1"} \ + "$somepath" | grep -v '^$' | tail -n 2) +fi + + +# Sample output (we want the last two lines): +# +# Linux 2.6.32-642.13.1.el6.x86_64 (centos68) 03/09/2017 _x86_64_ (6 CPU) +# +# avg-cpu: %user %nice %system %iowait %steal %idle +# 0.00 0.00 0.00 0.00 0.00 100.00 +# +# Device: rrqm/s wrqm/s r/s w/s rkB/s wkB/s avgrq-sz avgqu-sz await r_await w_await svctm %util +# sdb 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 +# + +# Get the column names +cols=$(echo "$out" | head -n 1) + +# Get the values and tab separate them to make them cut-able. +vals=$(echo "$out" | tail -n 1 | tr -s '[:space:]' '\t') + +i=0 +for col in $cols ; do + i=$((i+1)) + # Skip the first column since it's just the device name + if [ "$i" -eq 1 ]; then + continue + fi + + # Get i'th value + val=$(echo "$vals" | cut -f "$i") + echo "$col=$val" +done diff --git a/straper/db/public/zfs/etc-zfs/zpool.d/label b/straper/db/public/zfs/etc-zfs/zpool.d/label new file mode 100755 index 0000000..293effd --- /dev/null +++ b/straper/db/public/zfs/etc-zfs/zpool.d/label @@ -0,0 +1,92 @@ +#!/bin/sh +# +# Print some common lsblk values +# +# Any (lowercased) name symlinked to the lsblk script will be passed to lsblk +# as one of its --output names. Here's a partial list of --output names +# from the lsblk binary: +# +# Available columns (for --output): +# NAME device name +# KNAME internal kernel device name +# MAJ:MIN major:minor device number +# FSTYPE filesystem type +# MOUNTPOINT where the device is mounted +# LABEL filesystem LABEL +# UUID filesystem UUID +# RA read-ahead of the device +# RO read-only device +# RM removable device +# MODEL device identifier +# SIZE size of the device +# STATE state of the device +# OWNER user name +# GROUP group name +# MODE device node permissions +# ALIGNMENT alignment offset +# MIN-IO minimum I/O size +# OPT-IO optimal I/O size +# PHY-SEC physical sector size +# LOG-SEC logical sector size +# ROTA rotational device +# SCHED I/O scheduler name +# RQ-SIZE request queue size +# TYPE device type +# DISC-ALN discard alignment offset +# DISC-GRAN discard granularity +# DISC-MAX discard max bytes +# DISC-ZERO discard zeroes data +# +# If the script is run as just 'lsblk' then print out disk size, vendor, +# and model number. + + +helpstr=" +label: Show filesystem label. +model: Show disk model number. +size: Show the disk capacity. +vendor: Show the disk vendor. +lsblk: Show the disk size, vendor, and model number." + +script="${0##*/}" + +if [ "$1" = "-h" ] ; then + echo "$helpstr" | grep "$script:" | tr -s '\t' | cut -f 2- + exit +fi + +if [ "$script" = "lsblk" ] ; then + list="size vendor model" +else + list=$(echo "$script" | tr '[:upper:]' '[:lower:]') +fi + +# Sometimes, UPATH ends up /dev/(null). +# That should be corrected, but for now... +# shellcheck disable=SC2154 +if [ ! -b "$VDEV_UPATH" ]; then + somepath="${VDEV_PATH}" +else + somepath="${VDEV_UPATH}" +fi + +# Older versions of lsblk don't support all these values (like SERIAL). +for i in $list ; do + + # Special case: Looking up the size of a file-based vdev can't + # be done with lsblk. + if [ "$i" = "size" ] && [ -f "$somepath" ] ; then + size=$(du -h --apparent-size "$somepath" | cut -f 1) + echo "size=$size" + continue + fi + + + val="" + if val=$(eval "lsblk -dl -n -o $i $somepath 2>/dev/null") ; then + # Remove leading/trailing whitespace from value + val=$(echo "$val" | sed -e 's/^[[:space:]]*//' \ + -e 's/[[:space:]]*$//') + fi + echo "$i=$val" +done diff --git a/straper/db/public/zfs/etc-zfs/zpool.d/locate_led b/straper/db/public/zfs/etc-zfs/zpool.d/locate_led new file mode 100755 index 0000000..19ef92a --- /dev/null +++ b/straper/db/public/zfs/etc-zfs/zpool.d/locate_led @@ -0,0 +1,69 @@ +#!/bin/sh +# +# Print SCSI Enclosure Services (SES) info. The output is dependent on the name +# of the script/symlink used to call it. +# +helpstr=" +enc: Show disk enclosure w:x:y:z value. +slot: Show disk slot number as reported by the enclosure. +encdev: Show /dev/sg* device associated with the enclosure disk slot. +fault_led: Show value of the disk enclosure slot fault LED. +locate_led: Show value of the disk enclosure slot locate LED. +ses: Show disk's enc, enc device, slot, and fault/locate LED values." + +script="${0##*/}" +if [ "$1" = "-h" ] ; then + echo "$helpstr" | grep "$script:" | tr -s '\t' | cut -f 2- + exit +fi + +if [ "$script" = "ses" ] ; then + scripts='enc encdev slot fault_led locate_led' +else + scripts="$script" +fi + +for i in $scripts ; do + # shellcheck disable=SC2154 + if [ -z "$VDEV_ENC_SYSFS_PATH" ] ; then + echo "$i=" + continue + fi + + val="" + case $i in + enc) + if echo "$VDEV_ENC_SYSFS_PATH" | grep -q '/sys/bus/pci/slots' ; then + val="$VDEV_ENC_SYSFS_PATH" + else + val="$(ls """$VDEV_ENC_SYSFS_PATH/../../""" 2>/dev/null)" + fi + ;; + slot) + if echo "$VDEV_ENC_SYSFS_PATH" | grep -q '/sys/bus/pci/slots' ; then + val="$(basename """$VDEV_ENC_SYSFS_PATH""")" + else + val="$(cat """$VDEV_ENC_SYSFS_PATH/slot""" 2>/dev/null)" + fi + ;; + encdev) + val=$(ls "$VDEV_ENC_SYSFS_PATH/../device/scsi_generic" 2>/dev/null) + ;; + fault_led) + # JBODs fault LED is called 'fault', NVMe fault LED is called + # 'attention'. + if [ -f "$VDEV_ENC_SYSFS_PATH/fault" ] ; then + val=$(cat "$VDEV_ENC_SYSFS_PATH/fault" 2>/dev/null) + elif [ -f "$VDEV_ENC_SYSFS_PATH/attention" ] ; then + val=$(cat "$VDEV_ENC_SYSFS_PATH/attention" 2>/dev/null) + fi + ;; + locate_led) + val=$(cat "$VDEV_ENC_SYSFS_PATH/locate" 2>/dev/null) + ;; + *) + val=invalid + ;; + esac + echo "$i=$val" +done diff --git a/straper/db/public/zfs/etc-zfs/zpool.d/lsblk b/straper/db/public/zfs/etc-zfs/zpool.d/lsblk new file mode 100755 index 0000000..293effd --- /dev/null +++ b/straper/db/public/zfs/etc-zfs/zpool.d/lsblk @@ -0,0 +1,92 @@ +#!/bin/sh +# +# Print some common lsblk values +# +# Any (lowercased) name symlinked to the lsblk script will be passed to lsblk +# as one of its --output names. Here's a partial list of --output names +# from the lsblk binary: +# +# Available columns (for --output): +# NAME device name +# KNAME internal kernel device name +# MAJ:MIN major:minor device number +# FSTYPE filesystem type +# MOUNTPOINT where the device is mounted +# LABEL filesystem LABEL +# UUID filesystem UUID +# RA read-ahead of the device +# RO read-only device +# RM removable device +# MODEL device identifier +# SIZE size of the device +# STATE state of the device +# OWNER user name +# GROUP group name +# MODE device node permissions +# ALIGNMENT alignment offset +# MIN-IO minimum I/O size +# OPT-IO optimal I/O size +# PHY-SEC physical sector size +# LOG-SEC logical sector size +# ROTA rotational device +# SCHED I/O scheduler name +# RQ-SIZE request queue size +# TYPE device type +# DISC-ALN discard alignment offset +# DISC-GRAN discard granularity +# DISC-MAX discard max bytes +# DISC-ZERO discard zeroes data +# +# If the script is run as just 'lsblk' then print out disk size, vendor, +# and model number. + + +helpstr=" +label: Show filesystem label. +model: Show disk model number. +size: Show the disk capacity. +vendor: Show the disk vendor. +lsblk: Show the disk size, vendor, and model number." + +script="${0##*/}" + +if [ "$1" = "-h" ] ; then + echo "$helpstr" | grep "$script:" | tr -s '\t' | cut -f 2- + exit +fi + +if [ "$script" = "lsblk" ] ; then + list="size vendor model" +else + list=$(echo "$script" | tr '[:upper:]' '[:lower:]') +fi + +# Sometimes, UPATH ends up /dev/(null). +# That should be corrected, but for now... +# shellcheck disable=SC2154 +if [ ! -b "$VDEV_UPATH" ]; then + somepath="${VDEV_PATH}" +else + somepath="${VDEV_UPATH}" +fi + +# Older versions of lsblk don't support all these values (like SERIAL). +for i in $list ; do + + # Special case: Looking up the size of a file-based vdev can't + # be done with lsblk. + if [ "$i" = "size" ] && [ -f "$somepath" ] ; then + size=$(du -h --apparent-size "$somepath" | cut -f 1) + echo "size=$size" + continue + fi + + + val="" + if val=$(eval "lsblk -dl -n -o $i $somepath 2>/dev/null") ; then + # Remove leading/trailing whitespace from value + val=$(echo "$val" | sed -e 's/^[[:space:]]*//' \ + -e 's/[[:space:]]*$//') + fi + echo "$i=$val" +done diff --git a/straper/db/public/zfs/etc-zfs/zpool.d/media b/straper/db/public/zfs/etc-zfs/zpool.d/media new file mode 100755 index 0000000..095ac86 --- /dev/null +++ b/straper/db/public/zfs/etc-zfs/zpool.d/media @@ -0,0 +1,33 @@ +#!/bin/sh +# +# Print out the type of device +# + +if [ "$1" = "-h" ] ; then + echo "Show whether a vdev is a file, hdd, ssd, or iscsi." + exit +fi + +# shellcheck disable=SC2154 +if [ -b "$VDEV_UPATH" ]; then + device="${VDEV_UPATH##*/}" + read -r val 2>/dev/null < "/sys/block/$device/queue/rotational" + case "$val" in + 0) MEDIA="ssd" ;; + 1) MEDIA="hdd" ;; + *) MEDIA="invalid" ;; + esac + + vpd_pg83="/sys/block/$device/device/vpd_pg83" + if [ -f "$vpd_pg83" ]; then + if grep -q --binary "iqn." "$vpd_pg83"; then + MEDIA="iscsi" + fi + fi +else + if [ -f "$VDEV_UPATH" ]; then + MEDIA="file" + fi +fi + +echo "media=$MEDIA" diff --git a/straper/db/public/zfs/etc-zfs/zpool.d/model b/straper/db/public/zfs/etc-zfs/zpool.d/model new file mode 100755 index 0000000..293effd --- /dev/null +++ b/straper/db/public/zfs/etc-zfs/zpool.d/model @@ -0,0 +1,92 @@ +#!/bin/sh +# +# Print some common lsblk values +# +# Any (lowercased) name symlinked to the lsblk script will be passed to lsblk +# as one of its --output names. Here's a partial list of --output names +# from the lsblk binary: +# +# Available columns (for --output): +# NAME device name +# KNAME internal kernel device name +# MAJ:MIN major:minor device number +# FSTYPE filesystem type +# MOUNTPOINT where the device is mounted +# LABEL filesystem LABEL +# UUID filesystem UUID +# RA read-ahead of the device +# RO read-only device +# RM removable device +# MODEL device identifier +# SIZE size of the device +# STATE state of the device +# OWNER user name +# GROUP group name +# MODE device node permissions +# ALIGNMENT alignment offset +# MIN-IO minimum I/O size +# OPT-IO optimal I/O size +# PHY-SEC physical sector size +# LOG-SEC logical sector size +# ROTA rotational device +# SCHED I/O scheduler name +# RQ-SIZE request queue size +# TYPE device type +# DISC-ALN discard alignment offset +# DISC-GRAN discard granularity +# DISC-MAX discard max bytes +# DISC-ZERO discard zeroes data +# +# If the script is run as just 'lsblk' then print out disk size, vendor, +# and model number. + + +helpstr=" +label: Show filesystem label. +model: Show disk model number. +size: Show the disk capacity. +vendor: Show the disk vendor. +lsblk: Show the disk size, vendor, and model number." + +script="${0##*/}" + +if [ "$1" = "-h" ] ; then + echo "$helpstr" | grep "$script:" | tr -s '\t' | cut -f 2- + exit +fi + +if [ "$script" = "lsblk" ] ; then + list="size vendor model" +else + list=$(echo "$script" | tr '[:upper:]' '[:lower:]') +fi + +# Sometimes, UPATH ends up /dev/(null). +# That should be corrected, but for now... +# shellcheck disable=SC2154 +if [ ! -b "$VDEV_UPATH" ]; then + somepath="${VDEV_PATH}" +else + somepath="${VDEV_UPATH}" +fi + +# Older versions of lsblk don't support all these values (like SERIAL). +for i in $list ; do + + # Special case: Looking up the size of a file-based vdev can't + # be done with lsblk. + if [ "$i" = "size" ] && [ -f "$somepath" ] ; then + size=$(du -h --apparent-size "$somepath" | cut -f 1) + echo "size=$size" + continue + fi + + + val="" + if val=$(eval "lsblk -dl -n -o $i $somepath 2>/dev/null") ; then + # Remove leading/trailing whitespace from value + val=$(echo "$val" | sed -e 's/^[[:space:]]*//' \ + -e 's/[[:space:]]*$//') + fi + echo "$i=$val" +done diff --git a/straper/db/public/zfs/etc-zfs/zpool.d/nonmed b/straper/db/public/zfs/etc-zfs/zpool.d/nonmed new file mode 100755 index 0000000..8ad3e10 --- /dev/null +++ b/straper/db/public/zfs/etc-zfs/zpool.d/nonmed @@ -0,0 +1,249 @@ +#!/bin/sh +# +# Show SMART stats +# + +helpstr=" +smart: Show SMART temperature and error stats (specific to drive type) +smartx: Show SMART extended drive stats (specific to drive type). +temp: Show SMART drive temperature in celsius (all drives). +health: Show reported SMART status (all drives). +r_proc: Show SMART read GBytes processed over drive lifetime (SAS). +w_proc: Show SMART write GBytes processed over drive lifetime (SAS). +r_ucor: Show SMART read uncorrectable errors (SAS). +w_ucor: Show SMART write uncorrectable errors (SAS). +nonmed: Show SMART non-medium errors (SAS). +defect: Show SMART grown defect list (SAS). +hours_on: Show number of hours drive powered on (all drives). +realloc: Show SMART reallocated sectors count (ATA). +rep_ucor: Show SMART reported uncorrectable count (ATA). +cmd_to: Show SMART command timeout count (ATA). +pend_sec: Show SMART current pending sector count (ATA). +off_ucor: Show SMART offline uncorrectable errors (ATA). +ata_err: Show SMART ATA errors (ATA). +pwr_cyc: Show SMART power cycle count (ATA). +serial: Show disk serial number. +nvme_err: Show SMART NVMe errors (NVMe). +smart_test: Show SMART self-test results summary. +test_type: Show SMART self-test type (short, long... ). +test_status: Show SMART self-test status. +test_progress: Show SMART self-test percentage done. +test_ended: Show when the last SMART self-test ended (if supported). +" + +# Hack for developer testing +# +# If you set $samples to a directory containing smartctl output text files, +# we will use them instead of running smartctl on the vdevs. This can be +# useful if you want to test a bunch of different smartctl outputs. Also, if +# $samples is set, and additional 'file' column is added to the zpool output +# showing the filename. +samples= + +# get_filename_from_dir DIR +# +# Look in directory DIR and return a filename from it. The filename returned +# is chosen quasi-sequentially (based off our PID). This allows us to return +# a different filename every time this script is invoked (which we do for each +# vdev), without having to maintain state. +get_filename_from_dir() +{ + dir=$1 + pid="$$" + num_files=$(find "$dir" -maxdepth 1 -type f | wc -l) + mod=$((pid % num_files)) + i=0 + find "$dir" -type f -printf '%f\n' | while read -r file ; do + if [ "$mod" = "$i" ] ; then + echo "$file" + break + fi + i=$((i+1)) + done +} + +script="${0##*/}" + +if [ "$1" = "-h" ] ; then + echo "$helpstr" | grep "$script:" | tr -s '\t' | cut -f 2- + exit +fi + +# Sometimes, UPATH ends up /dev/(null). +# That should be corrected, but for now... +# shellcheck disable=SC2154 +if [ ! -b "$VDEV_UPATH" ]; then + somepath="${VDEV_PATH}" +else + somepath="${VDEV_UPATH}" +fi + +if [ -b "$somepath" ] && PATH="/usr/sbin:$PATH" command -v smartctl > /dev/null || [ -n "$samples" ] ; then + if [ -n "$samples" ] ; then + # cat a smartctl output text file instead of running smartctl + # on a vdev (only used for developer testing). + file=$(get_filename_from_dir "$samples") + echo "file=$file" + raw_out=$(cat "$samples/$file") + else + raw_out=$(sudo smartctl -a "$somepath") + fi + + # What kind of drive are we? Look for the right line in smartctl: + # + # SAS: + # Transport protocol: SAS + # + # SATA: + # ATA Version is: 8 + # + # NVMe: + # SMART/Health Information (NVMe Log 0xnn, NSID 0xnn) + # + out=$(echo "$raw_out" | awk ' +# SAS specific +/read:/{print "rrd="$4"\nr_cor="$5"\nr_proc="$7"\nr_ucor="$8} +/write:/{print "rwr="$4"\nw_cor="$5"\nw_proc="$7"\nw_ucor="$8} +/Non-medium error count/{print "nonmed="$4} +/Elements in grown defect list/{print "defect="$6} + +# SAS common +/SAS/{type="sas"} +/Drive Temperature:/{print "temp="$4} +# Status can be a long string, substitute spaces for '_' +/SMART Health Status:/{printf "health="; for(i=4;i<=NF-1;i++){printf "%s_", $i}; printf "%s\n", $i} +/number of hours powered up/{print "hours_on="$7; hours_on=int($7)} +/Serial number:/{print "serial="$3} + +# SATA specific +/Reallocated_Sector_Ct/{print "realloc="$10} +/Reported_Uncorrect/{print "rep_ucor="$10} +/Command_Timeout/{print "cmd_to="$10} +/Current_Pending_Sector/{print "pend_sec="$10} +/Offline_Uncorrectable/{print "off_ucor="$10} +/ATA Error Count:/{print "ata_err="$4} +/Power_Cycle_Count/{print "pwr_cyc="$10} + +# SATA common +/SATA/{type="sata"} +/Temperature_Celsius/{print "temp="$10} +/Airflow_Temperature_Cel/{print "temp="$10} +/Current Temperature:/{print "temp="$3} +/SMART overall-health self-assessment test result:/{print "health="$6} +/Power_On_Hours/{print "hours_on="$10; hours_on=int($10)} +/Serial Number:/{print "serial="$3} + +# NVMe common +/NVMe/{type="nvme"} +/Temperature:/{print "temp="$2} +/SMART overall-health self-assessment test result:/{print "health="$6} +/Power On Hours:/{gsub("[^0-9]","",$4); print "hours_on="$4} +/Serial Number:/{print "serial="$3} +/Power Cycles:/{print "pwr_cyc="$3} + +# NVMe specific +/Media and Data Integrity Errors:/{print "nvme_err="$6} + +# SMART self-test info +/Self-test execution status:/{progress=tolower($4)} # SAS +/SMART Self-test log/{test_seen=1} # SAS +/SMART Extended Self-test Log/{test_seen=1} # SATA +/# 1/{ + test_type=tolower($3"_"$4); + # Status could be one word ("Completed") or multiple ("Completed: read + # failure"). Look for the ":" to see if we need to grab more words. + + if ($5 ~ ":") + status=tolower($5""$6"_"$7) + else + status=tolower($5) + if (status=="self") + status="running"; + + if (type == "sas") { + hours=int($(NF-4)) + } else { + hours=int($(NF-1)) + # SATA reports percent remaining, rather than percent done + # Convert it to percent done. + progress=(100-int($(NF-2)))"%" + } + # When we int()-ify "hours", it converts stuff like "NOW" and "-" into + # 0. In those cases, set it to hours_on, so they will cancel out in + # the "hours_ago" calculation later on. + if (hours == 0) + hours=hours_on + + if (test_seen) { + print "test="hours_on + print "test_type="test_type + print "test_status="status + print "test_progress="progress + } + # Not all drives report hours_on + if (hours_on && hours) { + total_hours_ago=(hours_on-hours) + days_ago=int(total_hours_ago/24) + hours_ago=(total_hours_ago % 24) + if (days_ago != 0) + ago_str=days_ago"d" + if (hours_ago !=0) + ago_str=ago_str""hours_ago"h" + print "test_ended="ago_str + } +} + +END {print "type="type; ORS="\n"; print ""} +'); +fi +type=$(echo "$out" | grep '^type=' | cut -d '=' -f 2) + +# If type is not set by now, either we don't have a block device +# or smartctl failed. Either way, default to ATA and set $out to +# nothing. +if [ -z "$type" ]; then + type="sata" + out= +fi + +case $script in +smart) + # Print temperature plus common predictors of drive failure + if [ "$type" = "sas" ] ; then + scripts="temp|health|r_ucor|w_ucor" + elif [ "$type" = "sata" ] ; then + scripts="temp|health|ata_err|realloc|rep_ucor|cmd_to|pend_sec|off_ucor" + elif [ "$type" = "nvme" ] ; then + scripts="temp|health|nvme_err" + fi + ;; +smartx) + # Print some other interesting stats + if [ "$type" = "sas" ] ; then + scripts="hours_on|defect|nonmed|r_proc|w_proc" + elif [ "$type" = "sata" ] ; then + scripts="hours_on|pwr_cyc" + elif [ "$type" = "nvme" ] ; then + scripts="hours_on|pwr_cyc" + fi + ;; +smart_test) + scripts="test_type|test_status|test_progress|test_ended" + ;; +*) + scripts="$script" +esac + +with_vals=$(echo "$out" | grep -E "$scripts") +if [ -n "$with_vals" ]; then + echo "$with_vals" + without_vals=$(echo "$scripts" | tr '|' '\n' | + grep -v -E "$(echo "$with_vals" | + awk -F "=" '{print $1}')" | awk '{print $0"="}') +else + without_vals=$(echo "$scripts" | tr '|' '\n' | awk '{print $0"="}') +fi + +if [ -n "$without_vals" ]; then + echo "$without_vals" +fi diff --git a/straper/db/public/zfs/etc-zfs/zpool.d/nvme_err b/straper/db/public/zfs/etc-zfs/zpool.d/nvme_err new file mode 100755 index 0000000..8ad3e10 --- /dev/null +++ b/straper/db/public/zfs/etc-zfs/zpool.d/nvme_err @@ -0,0 +1,249 @@ +#!/bin/sh +# +# Show SMART stats +# + +helpstr=" +smart: Show SMART temperature and error stats (specific to drive type) +smartx: Show SMART extended drive stats (specific to drive type). +temp: Show SMART drive temperature in celsius (all drives). +health: Show reported SMART status (all drives). +r_proc: Show SMART read GBytes processed over drive lifetime (SAS). +w_proc: Show SMART write GBytes processed over drive lifetime (SAS). +r_ucor: Show SMART read uncorrectable errors (SAS). +w_ucor: Show SMART write uncorrectable errors (SAS). +nonmed: Show SMART non-medium errors (SAS). +defect: Show SMART grown defect list (SAS). +hours_on: Show number of hours drive powered on (all drives). +realloc: Show SMART reallocated sectors count (ATA). +rep_ucor: Show SMART reported uncorrectable count (ATA). +cmd_to: Show SMART command timeout count (ATA). +pend_sec: Show SMART current pending sector count (ATA). +off_ucor: Show SMART offline uncorrectable errors (ATA). +ata_err: Show SMART ATA errors (ATA). +pwr_cyc: Show SMART power cycle count (ATA). +serial: Show disk serial number. +nvme_err: Show SMART NVMe errors (NVMe). +smart_test: Show SMART self-test results summary. +test_type: Show SMART self-test type (short, long... ). +test_status: Show SMART self-test status. +test_progress: Show SMART self-test percentage done. +test_ended: Show when the last SMART self-test ended (if supported). +" + +# Hack for developer testing +# +# If you set $samples to a directory containing smartctl output text files, +# we will use them instead of running smartctl on the vdevs. This can be +# useful if you want to test a bunch of different smartctl outputs. Also, if +# $samples is set, and additional 'file' column is added to the zpool output +# showing the filename. +samples= + +# get_filename_from_dir DIR +# +# Look in directory DIR and return a filename from it. The filename returned +# is chosen quasi-sequentially (based off our PID). This allows us to return +# a different filename every time this script is invoked (which we do for each +# vdev), without having to maintain state. +get_filename_from_dir() +{ + dir=$1 + pid="$$" + num_files=$(find "$dir" -maxdepth 1 -type f | wc -l) + mod=$((pid % num_files)) + i=0 + find "$dir" -type f -printf '%f\n' | while read -r file ; do + if [ "$mod" = "$i" ] ; then + echo "$file" + break + fi + i=$((i+1)) + done +} + +script="${0##*/}" + +if [ "$1" = "-h" ] ; then + echo "$helpstr" | grep "$script:" | tr -s '\t' | cut -f 2- + exit +fi + +# Sometimes, UPATH ends up /dev/(null). +# That should be corrected, but for now... +# shellcheck disable=SC2154 +if [ ! -b "$VDEV_UPATH" ]; then + somepath="${VDEV_PATH}" +else + somepath="${VDEV_UPATH}" +fi + +if [ -b "$somepath" ] && PATH="/usr/sbin:$PATH" command -v smartctl > /dev/null || [ -n "$samples" ] ; then + if [ -n "$samples" ] ; then + # cat a smartctl output text file instead of running smartctl + # on a vdev (only used for developer testing). + file=$(get_filename_from_dir "$samples") + echo "file=$file" + raw_out=$(cat "$samples/$file") + else + raw_out=$(sudo smartctl -a "$somepath") + fi + + # What kind of drive are we? Look for the right line in smartctl: + # + # SAS: + # Transport protocol: SAS + # + # SATA: + # ATA Version is: 8 + # + # NVMe: + # SMART/Health Information (NVMe Log 0xnn, NSID 0xnn) + # + out=$(echo "$raw_out" | awk ' +# SAS specific +/read:/{print "rrd="$4"\nr_cor="$5"\nr_proc="$7"\nr_ucor="$8} +/write:/{print "rwr="$4"\nw_cor="$5"\nw_proc="$7"\nw_ucor="$8} +/Non-medium error count/{print "nonmed="$4} +/Elements in grown defect list/{print "defect="$6} + +# SAS common +/SAS/{type="sas"} +/Drive Temperature:/{print "temp="$4} +# Status can be a long string, substitute spaces for '_' +/SMART Health Status:/{printf "health="; for(i=4;i<=NF-1;i++){printf "%s_", $i}; printf "%s\n", $i} +/number of hours powered up/{print "hours_on="$7; hours_on=int($7)} +/Serial number:/{print "serial="$3} + +# SATA specific +/Reallocated_Sector_Ct/{print "realloc="$10} +/Reported_Uncorrect/{print "rep_ucor="$10} +/Command_Timeout/{print "cmd_to="$10} +/Current_Pending_Sector/{print "pend_sec="$10} +/Offline_Uncorrectable/{print "off_ucor="$10} +/ATA Error Count:/{print "ata_err="$4} +/Power_Cycle_Count/{print "pwr_cyc="$10} + +# SATA common +/SATA/{type="sata"} +/Temperature_Celsius/{print "temp="$10} +/Airflow_Temperature_Cel/{print "temp="$10} +/Current Temperature:/{print "temp="$3} +/SMART overall-health self-assessment test result:/{print "health="$6} +/Power_On_Hours/{print "hours_on="$10; hours_on=int($10)} +/Serial Number:/{print "serial="$3} + +# NVMe common +/NVMe/{type="nvme"} +/Temperature:/{print "temp="$2} +/SMART overall-health self-assessment test result:/{print "health="$6} +/Power On Hours:/{gsub("[^0-9]","",$4); print "hours_on="$4} +/Serial Number:/{print "serial="$3} +/Power Cycles:/{print "pwr_cyc="$3} + +# NVMe specific +/Media and Data Integrity Errors:/{print "nvme_err="$6} + +# SMART self-test info +/Self-test execution status:/{progress=tolower($4)} # SAS +/SMART Self-test log/{test_seen=1} # SAS +/SMART Extended Self-test Log/{test_seen=1} # SATA +/# 1/{ + test_type=tolower($3"_"$4); + # Status could be one word ("Completed") or multiple ("Completed: read + # failure"). Look for the ":" to see if we need to grab more words. + + if ($5 ~ ":") + status=tolower($5""$6"_"$7) + else + status=tolower($5) + if (status=="self") + status="running"; + + if (type == "sas") { + hours=int($(NF-4)) + } else { + hours=int($(NF-1)) + # SATA reports percent remaining, rather than percent done + # Convert it to percent done. + progress=(100-int($(NF-2)))"%" + } + # When we int()-ify "hours", it converts stuff like "NOW" and "-" into + # 0. In those cases, set it to hours_on, so they will cancel out in + # the "hours_ago" calculation later on. + if (hours == 0) + hours=hours_on + + if (test_seen) { + print "test="hours_on + print "test_type="test_type + print "test_status="status + print "test_progress="progress + } + # Not all drives report hours_on + if (hours_on && hours) { + total_hours_ago=(hours_on-hours) + days_ago=int(total_hours_ago/24) + hours_ago=(total_hours_ago % 24) + if (days_ago != 0) + ago_str=days_ago"d" + if (hours_ago !=0) + ago_str=ago_str""hours_ago"h" + print "test_ended="ago_str + } +} + +END {print "type="type; ORS="\n"; print ""} +'); +fi +type=$(echo "$out" | grep '^type=' | cut -d '=' -f 2) + +# If type is not set by now, either we don't have a block device +# or smartctl failed. Either way, default to ATA and set $out to +# nothing. +if [ -z "$type" ]; then + type="sata" + out= +fi + +case $script in +smart) + # Print temperature plus common predictors of drive failure + if [ "$type" = "sas" ] ; then + scripts="temp|health|r_ucor|w_ucor" + elif [ "$type" = "sata" ] ; then + scripts="temp|health|ata_err|realloc|rep_ucor|cmd_to|pend_sec|off_ucor" + elif [ "$type" = "nvme" ] ; then + scripts="temp|health|nvme_err" + fi + ;; +smartx) + # Print some other interesting stats + if [ "$type" = "sas" ] ; then + scripts="hours_on|defect|nonmed|r_proc|w_proc" + elif [ "$type" = "sata" ] ; then + scripts="hours_on|pwr_cyc" + elif [ "$type" = "nvme" ] ; then + scripts="hours_on|pwr_cyc" + fi + ;; +smart_test) + scripts="test_type|test_status|test_progress|test_ended" + ;; +*) + scripts="$script" +esac + +with_vals=$(echo "$out" | grep -E "$scripts") +if [ -n "$with_vals" ]; then + echo "$with_vals" + without_vals=$(echo "$scripts" | tr '|' '\n' | + grep -v -E "$(echo "$with_vals" | + awk -F "=" '{print $1}')" | awk '{print $0"="}') +else + without_vals=$(echo "$scripts" | tr '|' '\n' | awk '{print $0"="}') +fi + +if [ -n "$without_vals" ]; then + echo "$without_vals" +fi diff --git a/straper/db/public/zfs/etc-zfs/zpool.d/off_ucor b/straper/db/public/zfs/etc-zfs/zpool.d/off_ucor new file mode 100755 index 0000000..8ad3e10 --- /dev/null +++ b/straper/db/public/zfs/etc-zfs/zpool.d/off_ucor @@ -0,0 +1,249 @@ +#!/bin/sh +# +# Show SMART stats +# + +helpstr=" +smart: Show SMART temperature and error stats (specific to drive type) +smartx: Show SMART extended drive stats (specific to drive type). +temp: Show SMART drive temperature in celsius (all drives). +health: Show reported SMART status (all drives). +r_proc: Show SMART read GBytes processed over drive lifetime (SAS). +w_proc: Show SMART write GBytes processed over drive lifetime (SAS). +r_ucor: Show SMART read uncorrectable errors (SAS). +w_ucor: Show SMART write uncorrectable errors (SAS). +nonmed: Show SMART non-medium errors (SAS). +defect: Show SMART grown defect list (SAS). +hours_on: Show number of hours drive powered on (all drives). +realloc: Show SMART reallocated sectors count (ATA). +rep_ucor: Show SMART reported uncorrectable count (ATA). +cmd_to: Show SMART command timeout count (ATA). +pend_sec: Show SMART current pending sector count (ATA). +off_ucor: Show SMART offline uncorrectable errors (ATA). +ata_err: Show SMART ATA errors (ATA). +pwr_cyc: Show SMART power cycle count (ATA). +serial: Show disk serial number. +nvme_err: Show SMART NVMe errors (NVMe). +smart_test: Show SMART self-test results summary. +test_type: Show SMART self-test type (short, long... ). +test_status: Show SMART self-test status. +test_progress: Show SMART self-test percentage done. +test_ended: Show when the last SMART self-test ended (if supported). +" + +# Hack for developer testing +# +# If you set $samples to a directory containing smartctl output text files, +# we will use them instead of running smartctl on the vdevs. This can be +# useful if you want to test a bunch of different smartctl outputs. Also, if +# $samples is set, and additional 'file' column is added to the zpool output +# showing the filename. +samples= + +# get_filename_from_dir DIR +# +# Look in directory DIR and return a filename from it. The filename returned +# is chosen quasi-sequentially (based off our PID). This allows us to return +# a different filename every time this script is invoked (which we do for each +# vdev), without having to maintain state. +get_filename_from_dir() +{ + dir=$1 + pid="$$" + num_files=$(find "$dir" -maxdepth 1 -type f | wc -l) + mod=$((pid % num_files)) + i=0 + find "$dir" -type f -printf '%f\n' | while read -r file ; do + if [ "$mod" = "$i" ] ; then + echo "$file" + break + fi + i=$((i+1)) + done +} + +script="${0##*/}" + +if [ "$1" = "-h" ] ; then + echo "$helpstr" | grep "$script:" | tr -s '\t' | cut -f 2- + exit +fi + +# Sometimes, UPATH ends up /dev/(null). +# That should be corrected, but for now... +# shellcheck disable=SC2154 +if [ ! -b "$VDEV_UPATH" ]; then + somepath="${VDEV_PATH}" +else + somepath="${VDEV_UPATH}" +fi + +if [ -b "$somepath" ] && PATH="/usr/sbin:$PATH" command -v smartctl > /dev/null || [ -n "$samples" ] ; then + if [ -n "$samples" ] ; then + # cat a smartctl output text file instead of running smartctl + # on a vdev (only used for developer testing). + file=$(get_filename_from_dir "$samples") + echo "file=$file" + raw_out=$(cat "$samples/$file") + else + raw_out=$(sudo smartctl -a "$somepath") + fi + + # What kind of drive are we? Look for the right line in smartctl: + # + # SAS: + # Transport protocol: SAS + # + # SATA: + # ATA Version is: 8 + # + # NVMe: + # SMART/Health Information (NVMe Log 0xnn, NSID 0xnn) + # + out=$(echo "$raw_out" | awk ' +# SAS specific +/read:/{print "rrd="$4"\nr_cor="$5"\nr_proc="$7"\nr_ucor="$8} +/write:/{print "rwr="$4"\nw_cor="$5"\nw_proc="$7"\nw_ucor="$8} +/Non-medium error count/{print "nonmed="$4} +/Elements in grown defect list/{print "defect="$6} + +# SAS common +/SAS/{type="sas"} +/Drive Temperature:/{print "temp="$4} +# Status can be a long string, substitute spaces for '_' +/SMART Health Status:/{printf "health="; for(i=4;i<=NF-1;i++){printf "%s_", $i}; printf "%s\n", $i} +/number of hours powered up/{print "hours_on="$7; hours_on=int($7)} +/Serial number:/{print "serial="$3} + +# SATA specific +/Reallocated_Sector_Ct/{print "realloc="$10} +/Reported_Uncorrect/{print "rep_ucor="$10} +/Command_Timeout/{print "cmd_to="$10} +/Current_Pending_Sector/{print "pend_sec="$10} +/Offline_Uncorrectable/{print "off_ucor="$10} +/ATA Error Count:/{print "ata_err="$4} +/Power_Cycle_Count/{print "pwr_cyc="$10} + +# SATA common +/SATA/{type="sata"} +/Temperature_Celsius/{print "temp="$10} +/Airflow_Temperature_Cel/{print "temp="$10} +/Current Temperature:/{print "temp="$3} +/SMART overall-health self-assessment test result:/{print "health="$6} +/Power_On_Hours/{print "hours_on="$10; hours_on=int($10)} +/Serial Number:/{print "serial="$3} + +# NVMe common +/NVMe/{type="nvme"} +/Temperature:/{print "temp="$2} +/SMART overall-health self-assessment test result:/{print "health="$6} +/Power On Hours:/{gsub("[^0-9]","",$4); print "hours_on="$4} +/Serial Number:/{print "serial="$3} +/Power Cycles:/{print "pwr_cyc="$3} + +# NVMe specific +/Media and Data Integrity Errors:/{print "nvme_err="$6} + +# SMART self-test info +/Self-test execution status:/{progress=tolower($4)} # SAS +/SMART Self-test log/{test_seen=1} # SAS +/SMART Extended Self-test Log/{test_seen=1} # SATA +/# 1/{ + test_type=tolower($3"_"$4); + # Status could be one word ("Completed") or multiple ("Completed: read + # failure"). Look for the ":" to see if we need to grab more words. + + if ($5 ~ ":") + status=tolower($5""$6"_"$7) + else + status=tolower($5) + if (status=="self") + status="running"; + + if (type == "sas") { + hours=int($(NF-4)) + } else { + hours=int($(NF-1)) + # SATA reports percent remaining, rather than percent done + # Convert it to percent done. + progress=(100-int($(NF-2)))"%" + } + # When we int()-ify "hours", it converts stuff like "NOW" and "-" into + # 0. In those cases, set it to hours_on, so they will cancel out in + # the "hours_ago" calculation later on. + if (hours == 0) + hours=hours_on + + if (test_seen) { + print "test="hours_on + print "test_type="test_type + print "test_status="status + print "test_progress="progress + } + # Not all drives report hours_on + if (hours_on && hours) { + total_hours_ago=(hours_on-hours) + days_ago=int(total_hours_ago/24) + hours_ago=(total_hours_ago % 24) + if (days_ago != 0) + ago_str=days_ago"d" + if (hours_ago !=0) + ago_str=ago_str""hours_ago"h" + print "test_ended="ago_str + } +} + +END {print "type="type; ORS="\n"; print ""} +'); +fi +type=$(echo "$out" | grep '^type=' | cut -d '=' -f 2) + +# If type is not set by now, either we don't have a block device +# or smartctl failed. Either way, default to ATA and set $out to +# nothing. +if [ -z "$type" ]; then + type="sata" + out= +fi + +case $script in +smart) + # Print temperature plus common predictors of drive failure + if [ "$type" = "sas" ] ; then + scripts="temp|health|r_ucor|w_ucor" + elif [ "$type" = "sata" ] ; then + scripts="temp|health|ata_err|realloc|rep_ucor|cmd_to|pend_sec|off_ucor" + elif [ "$type" = "nvme" ] ; then + scripts="temp|health|nvme_err" + fi + ;; +smartx) + # Print some other interesting stats + if [ "$type" = "sas" ] ; then + scripts="hours_on|defect|nonmed|r_proc|w_proc" + elif [ "$type" = "sata" ] ; then + scripts="hours_on|pwr_cyc" + elif [ "$type" = "nvme" ] ; then + scripts="hours_on|pwr_cyc" + fi + ;; +smart_test) + scripts="test_type|test_status|test_progress|test_ended" + ;; +*) + scripts="$script" +esac + +with_vals=$(echo "$out" | grep -E "$scripts") +if [ -n "$with_vals" ]; then + echo "$with_vals" + without_vals=$(echo "$scripts" | tr '|' '\n' | + grep -v -E "$(echo "$with_vals" | + awk -F "=" '{print $1}')" | awk '{print $0"="}') +else + without_vals=$(echo "$scripts" | tr '|' '\n' | awk '{print $0"="}') +fi + +if [ -n "$without_vals" ]; then + echo "$without_vals" +fi diff --git a/straper/db/public/zfs/etc-zfs/zpool.d/pend_sec b/straper/db/public/zfs/etc-zfs/zpool.d/pend_sec new file mode 100755 index 0000000..8ad3e10 --- /dev/null +++ b/straper/db/public/zfs/etc-zfs/zpool.d/pend_sec @@ -0,0 +1,249 @@ +#!/bin/sh +# +# Show SMART stats +# + +helpstr=" +smart: Show SMART temperature and error stats (specific to drive type) +smartx: Show SMART extended drive stats (specific to drive type). +temp: Show SMART drive temperature in celsius (all drives). +health: Show reported SMART status (all drives). +r_proc: Show SMART read GBytes processed over drive lifetime (SAS). +w_proc: Show SMART write GBytes processed over drive lifetime (SAS). +r_ucor: Show SMART read uncorrectable errors (SAS). +w_ucor: Show SMART write uncorrectable errors (SAS). +nonmed: Show SMART non-medium errors (SAS). +defect: Show SMART grown defect list (SAS). +hours_on: Show number of hours drive powered on (all drives). +realloc: Show SMART reallocated sectors count (ATA). +rep_ucor: Show SMART reported uncorrectable count (ATA). +cmd_to: Show SMART command timeout count (ATA). +pend_sec: Show SMART current pending sector count (ATA). +off_ucor: Show SMART offline uncorrectable errors (ATA). +ata_err: Show SMART ATA errors (ATA). +pwr_cyc: Show SMART power cycle count (ATA). +serial: Show disk serial number. +nvme_err: Show SMART NVMe errors (NVMe). +smart_test: Show SMART self-test results summary. +test_type: Show SMART self-test type (short, long... ). +test_status: Show SMART self-test status. +test_progress: Show SMART self-test percentage done. +test_ended: Show when the last SMART self-test ended (if supported). +" + +# Hack for developer testing +# +# If you set $samples to a directory containing smartctl output text files, +# we will use them instead of running smartctl on the vdevs. This can be +# useful if you want to test a bunch of different smartctl outputs. Also, if +# $samples is set, and additional 'file' column is added to the zpool output +# showing the filename. +samples= + +# get_filename_from_dir DIR +# +# Look in directory DIR and return a filename from it. The filename returned +# is chosen quasi-sequentially (based off our PID). This allows us to return +# a different filename every time this script is invoked (which we do for each +# vdev), without having to maintain state. +get_filename_from_dir() +{ + dir=$1 + pid="$$" + num_files=$(find "$dir" -maxdepth 1 -type f | wc -l) + mod=$((pid % num_files)) + i=0 + find "$dir" -type f -printf '%f\n' | while read -r file ; do + if [ "$mod" = "$i" ] ; then + echo "$file" + break + fi + i=$((i+1)) + done +} + +script="${0##*/}" + +if [ "$1" = "-h" ] ; then + echo "$helpstr" | grep "$script:" | tr -s '\t' | cut -f 2- + exit +fi + +# Sometimes, UPATH ends up /dev/(null). +# That should be corrected, but for now... +# shellcheck disable=SC2154 +if [ ! -b "$VDEV_UPATH" ]; then + somepath="${VDEV_PATH}" +else + somepath="${VDEV_UPATH}" +fi + +if [ -b "$somepath" ] && PATH="/usr/sbin:$PATH" command -v smartctl > /dev/null || [ -n "$samples" ] ; then + if [ -n "$samples" ] ; then + # cat a smartctl output text file instead of running smartctl + # on a vdev (only used for developer testing). + file=$(get_filename_from_dir "$samples") + echo "file=$file" + raw_out=$(cat "$samples/$file") + else + raw_out=$(sudo smartctl -a "$somepath") + fi + + # What kind of drive are we? Look for the right line in smartctl: + # + # SAS: + # Transport protocol: SAS + # + # SATA: + # ATA Version is: 8 + # + # NVMe: + # SMART/Health Information (NVMe Log 0xnn, NSID 0xnn) + # + out=$(echo "$raw_out" | awk ' +# SAS specific +/read:/{print "rrd="$4"\nr_cor="$5"\nr_proc="$7"\nr_ucor="$8} +/write:/{print "rwr="$4"\nw_cor="$5"\nw_proc="$7"\nw_ucor="$8} +/Non-medium error count/{print "nonmed="$4} +/Elements in grown defect list/{print "defect="$6} + +# SAS common +/SAS/{type="sas"} +/Drive Temperature:/{print "temp="$4} +# Status can be a long string, substitute spaces for '_' +/SMART Health Status:/{printf "health="; for(i=4;i<=NF-1;i++){printf "%s_", $i}; printf "%s\n", $i} +/number of hours powered up/{print "hours_on="$7; hours_on=int($7)} +/Serial number:/{print "serial="$3} + +# SATA specific +/Reallocated_Sector_Ct/{print "realloc="$10} +/Reported_Uncorrect/{print "rep_ucor="$10} +/Command_Timeout/{print "cmd_to="$10} +/Current_Pending_Sector/{print "pend_sec="$10} +/Offline_Uncorrectable/{print "off_ucor="$10} +/ATA Error Count:/{print "ata_err="$4} +/Power_Cycle_Count/{print "pwr_cyc="$10} + +# SATA common +/SATA/{type="sata"} +/Temperature_Celsius/{print "temp="$10} +/Airflow_Temperature_Cel/{print "temp="$10} +/Current Temperature:/{print "temp="$3} +/SMART overall-health self-assessment test result:/{print "health="$6} +/Power_On_Hours/{print "hours_on="$10; hours_on=int($10)} +/Serial Number:/{print "serial="$3} + +# NVMe common +/NVMe/{type="nvme"} +/Temperature:/{print "temp="$2} +/SMART overall-health self-assessment test result:/{print "health="$6} +/Power On Hours:/{gsub("[^0-9]","",$4); print "hours_on="$4} +/Serial Number:/{print "serial="$3} +/Power Cycles:/{print "pwr_cyc="$3} + +# NVMe specific +/Media and Data Integrity Errors:/{print "nvme_err="$6} + +# SMART self-test info +/Self-test execution status:/{progress=tolower($4)} # SAS +/SMART Self-test log/{test_seen=1} # SAS +/SMART Extended Self-test Log/{test_seen=1} # SATA +/# 1/{ + test_type=tolower($3"_"$4); + # Status could be one word ("Completed") or multiple ("Completed: read + # failure"). Look for the ":" to see if we need to grab more words. + + if ($5 ~ ":") + status=tolower($5""$6"_"$7) + else + status=tolower($5) + if (status=="self") + status="running"; + + if (type == "sas") { + hours=int($(NF-4)) + } else { + hours=int($(NF-1)) + # SATA reports percent remaining, rather than percent done + # Convert it to percent done. + progress=(100-int($(NF-2)))"%" + } + # When we int()-ify "hours", it converts stuff like "NOW" and "-" into + # 0. In those cases, set it to hours_on, so they will cancel out in + # the "hours_ago" calculation later on. + if (hours == 0) + hours=hours_on + + if (test_seen) { + print "test="hours_on + print "test_type="test_type + print "test_status="status + print "test_progress="progress + } + # Not all drives report hours_on + if (hours_on && hours) { + total_hours_ago=(hours_on-hours) + days_ago=int(total_hours_ago/24) + hours_ago=(total_hours_ago % 24) + if (days_ago != 0) + ago_str=days_ago"d" + if (hours_ago !=0) + ago_str=ago_str""hours_ago"h" + print "test_ended="ago_str + } +} + +END {print "type="type; ORS="\n"; print ""} +'); +fi +type=$(echo "$out" | grep '^type=' | cut -d '=' -f 2) + +# If type is not set by now, either we don't have a block device +# or smartctl failed. Either way, default to ATA and set $out to +# nothing. +if [ -z "$type" ]; then + type="sata" + out= +fi + +case $script in +smart) + # Print temperature plus common predictors of drive failure + if [ "$type" = "sas" ] ; then + scripts="temp|health|r_ucor|w_ucor" + elif [ "$type" = "sata" ] ; then + scripts="temp|health|ata_err|realloc|rep_ucor|cmd_to|pend_sec|off_ucor" + elif [ "$type" = "nvme" ] ; then + scripts="temp|health|nvme_err" + fi + ;; +smartx) + # Print some other interesting stats + if [ "$type" = "sas" ] ; then + scripts="hours_on|defect|nonmed|r_proc|w_proc" + elif [ "$type" = "sata" ] ; then + scripts="hours_on|pwr_cyc" + elif [ "$type" = "nvme" ] ; then + scripts="hours_on|pwr_cyc" + fi + ;; +smart_test) + scripts="test_type|test_status|test_progress|test_ended" + ;; +*) + scripts="$script" +esac + +with_vals=$(echo "$out" | grep -E "$scripts") +if [ -n "$with_vals" ]; then + echo "$with_vals" + without_vals=$(echo "$scripts" | tr '|' '\n' | + grep -v -E "$(echo "$with_vals" | + awk -F "=" '{print $1}')" | awk '{print $0"="}') +else + without_vals=$(echo "$scripts" | tr '|' '\n' | awk '{print $0"="}') +fi + +if [ -n "$without_vals" ]; then + echo "$without_vals" +fi diff --git a/straper/db/public/zfs/etc-zfs/zpool.d/pwr_cyc b/straper/db/public/zfs/etc-zfs/zpool.d/pwr_cyc new file mode 100755 index 0000000..8ad3e10 --- /dev/null +++ b/straper/db/public/zfs/etc-zfs/zpool.d/pwr_cyc @@ -0,0 +1,249 @@ +#!/bin/sh +# +# Show SMART stats +# + +helpstr=" +smart: Show SMART temperature and error stats (specific to drive type) +smartx: Show SMART extended drive stats (specific to drive type). +temp: Show SMART drive temperature in celsius (all drives). +health: Show reported SMART status (all drives). +r_proc: Show SMART read GBytes processed over drive lifetime (SAS). +w_proc: Show SMART write GBytes processed over drive lifetime (SAS). +r_ucor: Show SMART read uncorrectable errors (SAS). +w_ucor: Show SMART write uncorrectable errors (SAS). +nonmed: Show SMART non-medium errors (SAS). +defect: Show SMART grown defect list (SAS). +hours_on: Show number of hours drive powered on (all drives). +realloc: Show SMART reallocated sectors count (ATA). +rep_ucor: Show SMART reported uncorrectable count (ATA). +cmd_to: Show SMART command timeout count (ATA). +pend_sec: Show SMART current pending sector count (ATA). +off_ucor: Show SMART offline uncorrectable errors (ATA). +ata_err: Show SMART ATA errors (ATA). +pwr_cyc: Show SMART power cycle count (ATA). +serial: Show disk serial number. +nvme_err: Show SMART NVMe errors (NVMe). +smart_test: Show SMART self-test results summary. +test_type: Show SMART self-test type (short, long... ). +test_status: Show SMART self-test status. +test_progress: Show SMART self-test percentage done. +test_ended: Show when the last SMART self-test ended (if supported). +" + +# Hack for developer testing +# +# If you set $samples to a directory containing smartctl output text files, +# we will use them instead of running smartctl on the vdevs. This can be +# useful if you want to test a bunch of different smartctl outputs. Also, if +# $samples is set, and additional 'file' column is added to the zpool output +# showing the filename. +samples= + +# get_filename_from_dir DIR +# +# Look in directory DIR and return a filename from it. The filename returned +# is chosen quasi-sequentially (based off our PID). This allows us to return +# a different filename every time this script is invoked (which we do for each +# vdev), without having to maintain state. +get_filename_from_dir() +{ + dir=$1 + pid="$$" + num_files=$(find "$dir" -maxdepth 1 -type f | wc -l) + mod=$((pid % num_files)) + i=0 + find "$dir" -type f -printf '%f\n' | while read -r file ; do + if [ "$mod" = "$i" ] ; then + echo "$file" + break + fi + i=$((i+1)) + done +} + +script="${0##*/}" + +if [ "$1" = "-h" ] ; then + echo "$helpstr" | grep "$script:" | tr -s '\t' | cut -f 2- + exit +fi + +# Sometimes, UPATH ends up /dev/(null). +# That should be corrected, but for now... +# shellcheck disable=SC2154 +if [ ! -b "$VDEV_UPATH" ]; then + somepath="${VDEV_PATH}" +else + somepath="${VDEV_UPATH}" +fi + +if [ -b "$somepath" ] && PATH="/usr/sbin:$PATH" command -v smartctl > /dev/null || [ -n "$samples" ] ; then + if [ -n "$samples" ] ; then + # cat a smartctl output text file instead of running smartctl + # on a vdev (only used for developer testing). + file=$(get_filename_from_dir "$samples") + echo "file=$file" + raw_out=$(cat "$samples/$file") + else + raw_out=$(sudo smartctl -a "$somepath") + fi + + # What kind of drive are we? Look for the right line in smartctl: + # + # SAS: + # Transport protocol: SAS + # + # SATA: + # ATA Version is: 8 + # + # NVMe: + # SMART/Health Information (NVMe Log 0xnn, NSID 0xnn) + # + out=$(echo "$raw_out" | awk ' +# SAS specific +/read:/{print "rrd="$4"\nr_cor="$5"\nr_proc="$7"\nr_ucor="$8} +/write:/{print "rwr="$4"\nw_cor="$5"\nw_proc="$7"\nw_ucor="$8} +/Non-medium error count/{print "nonmed="$4} +/Elements in grown defect list/{print "defect="$6} + +# SAS common +/SAS/{type="sas"} +/Drive Temperature:/{print "temp="$4} +# Status can be a long string, substitute spaces for '_' +/SMART Health Status:/{printf "health="; for(i=4;i<=NF-1;i++){printf "%s_", $i}; printf "%s\n", $i} +/number of hours powered up/{print "hours_on="$7; hours_on=int($7)} +/Serial number:/{print "serial="$3} + +# SATA specific +/Reallocated_Sector_Ct/{print "realloc="$10} +/Reported_Uncorrect/{print "rep_ucor="$10} +/Command_Timeout/{print "cmd_to="$10} +/Current_Pending_Sector/{print "pend_sec="$10} +/Offline_Uncorrectable/{print "off_ucor="$10} +/ATA Error Count:/{print "ata_err="$4} +/Power_Cycle_Count/{print "pwr_cyc="$10} + +# SATA common +/SATA/{type="sata"} +/Temperature_Celsius/{print "temp="$10} +/Airflow_Temperature_Cel/{print "temp="$10} +/Current Temperature:/{print "temp="$3} +/SMART overall-health self-assessment test result:/{print "health="$6} +/Power_On_Hours/{print "hours_on="$10; hours_on=int($10)} +/Serial Number:/{print "serial="$3} + +# NVMe common +/NVMe/{type="nvme"} +/Temperature:/{print "temp="$2} +/SMART overall-health self-assessment test result:/{print "health="$6} +/Power On Hours:/{gsub("[^0-9]","",$4); print "hours_on="$4} +/Serial Number:/{print "serial="$3} +/Power Cycles:/{print "pwr_cyc="$3} + +# NVMe specific +/Media and Data Integrity Errors:/{print "nvme_err="$6} + +# SMART self-test info +/Self-test execution status:/{progress=tolower($4)} # SAS +/SMART Self-test log/{test_seen=1} # SAS +/SMART Extended Self-test Log/{test_seen=1} # SATA +/# 1/{ + test_type=tolower($3"_"$4); + # Status could be one word ("Completed") or multiple ("Completed: read + # failure"). Look for the ":" to see if we need to grab more words. + + if ($5 ~ ":") + status=tolower($5""$6"_"$7) + else + status=tolower($5) + if (status=="self") + status="running"; + + if (type == "sas") { + hours=int($(NF-4)) + } else { + hours=int($(NF-1)) + # SATA reports percent remaining, rather than percent done + # Convert it to percent done. + progress=(100-int($(NF-2)))"%" + } + # When we int()-ify "hours", it converts stuff like "NOW" and "-" into + # 0. In those cases, set it to hours_on, so they will cancel out in + # the "hours_ago" calculation later on. + if (hours == 0) + hours=hours_on + + if (test_seen) { + print "test="hours_on + print "test_type="test_type + print "test_status="status + print "test_progress="progress + } + # Not all drives report hours_on + if (hours_on && hours) { + total_hours_ago=(hours_on-hours) + days_ago=int(total_hours_ago/24) + hours_ago=(total_hours_ago % 24) + if (days_ago != 0) + ago_str=days_ago"d" + if (hours_ago !=0) + ago_str=ago_str""hours_ago"h" + print "test_ended="ago_str + } +} + +END {print "type="type; ORS="\n"; print ""} +'); +fi +type=$(echo "$out" | grep '^type=' | cut -d '=' -f 2) + +# If type is not set by now, either we don't have a block device +# or smartctl failed. Either way, default to ATA and set $out to +# nothing. +if [ -z "$type" ]; then + type="sata" + out= +fi + +case $script in +smart) + # Print temperature plus common predictors of drive failure + if [ "$type" = "sas" ] ; then + scripts="temp|health|r_ucor|w_ucor" + elif [ "$type" = "sata" ] ; then + scripts="temp|health|ata_err|realloc|rep_ucor|cmd_to|pend_sec|off_ucor" + elif [ "$type" = "nvme" ] ; then + scripts="temp|health|nvme_err" + fi + ;; +smartx) + # Print some other interesting stats + if [ "$type" = "sas" ] ; then + scripts="hours_on|defect|nonmed|r_proc|w_proc" + elif [ "$type" = "sata" ] ; then + scripts="hours_on|pwr_cyc" + elif [ "$type" = "nvme" ] ; then + scripts="hours_on|pwr_cyc" + fi + ;; +smart_test) + scripts="test_type|test_status|test_progress|test_ended" + ;; +*) + scripts="$script" +esac + +with_vals=$(echo "$out" | grep -E "$scripts") +if [ -n "$with_vals" ]; then + echo "$with_vals" + without_vals=$(echo "$scripts" | tr '|' '\n' | + grep -v -E "$(echo "$with_vals" | + awk -F "=" '{print $1}')" | awk '{print $0"="}') +else + without_vals=$(echo "$scripts" | tr '|' '\n' | awk '{print $0"="}') +fi + +if [ -n "$without_vals" ]; then + echo "$without_vals" +fi diff --git a/straper/db/public/zfs/etc-zfs/zpool.d/r_proc b/straper/db/public/zfs/etc-zfs/zpool.d/r_proc new file mode 100755 index 0000000..8ad3e10 --- /dev/null +++ b/straper/db/public/zfs/etc-zfs/zpool.d/r_proc @@ -0,0 +1,249 @@ +#!/bin/sh +# +# Show SMART stats +# + +helpstr=" +smart: Show SMART temperature and error stats (specific to drive type) +smartx: Show SMART extended drive stats (specific to drive type). +temp: Show SMART drive temperature in celsius (all drives). +health: Show reported SMART status (all drives). +r_proc: Show SMART read GBytes processed over drive lifetime (SAS). +w_proc: Show SMART write GBytes processed over drive lifetime (SAS). +r_ucor: Show SMART read uncorrectable errors (SAS). +w_ucor: Show SMART write uncorrectable errors (SAS). +nonmed: Show SMART non-medium errors (SAS). +defect: Show SMART grown defect list (SAS). +hours_on: Show number of hours drive powered on (all drives). +realloc: Show SMART reallocated sectors count (ATA). +rep_ucor: Show SMART reported uncorrectable count (ATA). +cmd_to: Show SMART command timeout count (ATA). +pend_sec: Show SMART current pending sector count (ATA). +off_ucor: Show SMART offline uncorrectable errors (ATA). +ata_err: Show SMART ATA errors (ATA). +pwr_cyc: Show SMART power cycle count (ATA). +serial: Show disk serial number. +nvme_err: Show SMART NVMe errors (NVMe). +smart_test: Show SMART self-test results summary. +test_type: Show SMART self-test type (short, long... ). +test_status: Show SMART self-test status. +test_progress: Show SMART self-test percentage done. +test_ended: Show when the last SMART self-test ended (if supported). +" + +# Hack for developer testing +# +# If you set $samples to a directory containing smartctl output text files, +# we will use them instead of running smartctl on the vdevs. This can be +# useful if you want to test a bunch of different smartctl outputs. Also, if +# $samples is set, and additional 'file' column is added to the zpool output +# showing the filename. +samples= + +# get_filename_from_dir DIR +# +# Look in directory DIR and return a filename from it. The filename returned +# is chosen quasi-sequentially (based off our PID). This allows us to return +# a different filename every time this script is invoked (which we do for each +# vdev), without having to maintain state. +get_filename_from_dir() +{ + dir=$1 + pid="$$" + num_files=$(find "$dir" -maxdepth 1 -type f | wc -l) + mod=$((pid % num_files)) + i=0 + find "$dir" -type f -printf '%f\n' | while read -r file ; do + if [ "$mod" = "$i" ] ; then + echo "$file" + break + fi + i=$((i+1)) + done +} + +script="${0##*/}" + +if [ "$1" = "-h" ] ; then + echo "$helpstr" | grep "$script:" | tr -s '\t' | cut -f 2- + exit +fi + +# Sometimes, UPATH ends up /dev/(null). +# That should be corrected, but for now... +# shellcheck disable=SC2154 +if [ ! -b "$VDEV_UPATH" ]; then + somepath="${VDEV_PATH}" +else + somepath="${VDEV_UPATH}" +fi + +if [ -b "$somepath" ] && PATH="/usr/sbin:$PATH" command -v smartctl > /dev/null || [ -n "$samples" ] ; then + if [ -n "$samples" ] ; then + # cat a smartctl output text file instead of running smartctl + # on a vdev (only used for developer testing). + file=$(get_filename_from_dir "$samples") + echo "file=$file" + raw_out=$(cat "$samples/$file") + else + raw_out=$(sudo smartctl -a "$somepath") + fi + + # What kind of drive are we? Look for the right line in smartctl: + # + # SAS: + # Transport protocol: SAS + # + # SATA: + # ATA Version is: 8 + # + # NVMe: + # SMART/Health Information (NVMe Log 0xnn, NSID 0xnn) + # + out=$(echo "$raw_out" | awk ' +# SAS specific +/read:/{print "rrd="$4"\nr_cor="$5"\nr_proc="$7"\nr_ucor="$8} +/write:/{print "rwr="$4"\nw_cor="$5"\nw_proc="$7"\nw_ucor="$8} +/Non-medium error count/{print "nonmed="$4} +/Elements in grown defect list/{print "defect="$6} + +# SAS common +/SAS/{type="sas"} +/Drive Temperature:/{print "temp="$4} +# Status can be a long string, substitute spaces for '_' +/SMART Health Status:/{printf "health="; for(i=4;i<=NF-1;i++){printf "%s_", $i}; printf "%s\n", $i} +/number of hours powered up/{print "hours_on="$7; hours_on=int($7)} +/Serial number:/{print "serial="$3} + +# SATA specific +/Reallocated_Sector_Ct/{print "realloc="$10} +/Reported_Uncorrect/{print "rep_ucor="$10} +/Command_Timeout/{print "cmd_to="$10} +/Current_Pending_Sector/{print "pend_sec="$10} +/Offline_Uncorrectable/{print "off_ucor="$10} +/ATA Error Count:/{print "ata_err="$4} +/Power_Cycle_Count/{print "pwr_cyc="$10} + +# SATA common +/SATA/{type="sata"} +/Temperature_Celsius/{print "temp="$10} +/Airflow_Temperature_Cel/{print "temp="$10} +/Current Temperature:/{print "temp="$3} +/SMART overall-health self-assessment test result:/{print "health="$6} +/Power_On_Hours/{print "hours_on="$10; hours_on=int($10)} +/Serial Number:/{print "serial="$3} + +# NVMe common +/NVMe/{type="nvme"} +/Temperature:/{print "temp="$2} +/SMART overall-health self-assessment test result:/{print "health="$6} +/Power On Hours:/{gsub("[^0-9]","",$4); print "hours_on="$4} +/Serial Number:/{print "serial="$3} +/Power Cycles:/{print "pwr_cyc="$3} + +# NVMe specific +/Media and Data Integrity Errors:/{print "nvme_err="$6} + +# SMART self-test info +/Self-test execution status:/{progress=tolower($4)} # SAS +/SMART Self-test log/{test_seen=1} # SAS +/SMART Extended Self-test Log/{test_seen=1} # SATA +/# 1/{ + test_type=tolower($3"_"$4); + # Status could be one word ("Completed") or multiple ("Completed: read + # failure"). Look for the ":" to see if we need to grab more words. + + if ($5 ~ ":") + status=tolower($5""$6"_"$7) + else + status=tolower($5) + if (status=="self") + status="running"; + + if (type == "sas") { + hours=int($(NF-4)) + } else { + hours=int($(NF-1)) + # SATA reports percent remaining, rather than percent done + # Convert it to percent done. + progress=(100-int($(NF-2)))"%" + } + # When we int()-ify "hours", it converts stuff like "NOW" and "-" into + # 0. In those cases, set it to hours_on, so they will cancel out in + # the "hours_ago" calculation later on. + if (hours == 0) + hours=hours_on + + if (test_seen) { + print "test="hours_on + print "test_type="test_type + print "test_status="status + print "test_progress="progress + } + # Not all drives report hours_on + if (hours_on && hours) { + total_hours_ago=(hours_on-hours) + days_ago=int(total_hours_ago/24) + hours_ago=(total_hours_ago % 24) + if (days_ago != 0) + ago_str=days_ago"d" + if (hours_ago !=0) + ago_str=ago_str""hours_ago"h" + print "test_ended="ago_str + } +} + +END {print "type="type; ORS="\n"; print ""} +'); +fi +type=$(echo "$out" | grep '^type=' | cut -d '=' -f 2) + +# If type is not set by now, either we don't have a block device +# or smartctl failed. Either way, default to ATA and set $out to +# nothing. +if [ -z "$type" ]; then + type="sata" + out= +fi + +case $script in +smart) + # Print temperature plus common predictors of drive failure + if [ "$type" = "sas" ] ; then + scripts="temp|health|r_ucor|w_ucor" + elif [ "$type" = "sata" ] ; then + scripts="temp|health|ata_err|realloc|rep_ucor|cmd_to|pend_sec|off_ucor" + elif [ "$type" = "nvme" ] ; then + scripts="temp|health|nvme_err" + fi + ;; +smartx) + # Print some other interesting stats + if [ "$type" = "sas" ] ; then + scripts="hours_on|defect|nonmed|r_proc|w_proc" + elif [ "$type" = "sata" ] ; then + scripts="hours_on|pwr_cyc" + elif [ "$type" = "nvme" ] ; then + scripts="hours_on|pwr_cyc" + fi + ;; +smart_test) + scripts="test_type|test_status|test_progress|test_ended" + ;; +*) + scripts="$script" +esac + +with_vals=$(echo "$out" | grep -E "$scripts") +if [ -n "$with_vals" ]; then + echo "$with_vals" + without_vals=$(echo "$scripts" | tr '|' '\n' | + grep -v -E "$(echo "$with_vals" | + awk -F "=" '{print $1}')" | awk '{print $0"="}') +else + without_vals=$(echo "$scripts" | tr '|' '\n' | awk '{print $0"="}') +fi + +if [ -n "$without_vals" ]; then + echo "$without_vals" +fi diff --git a/straper/db/public/zfs/etc-zfs/zpool.d/r_ucor b/straper/db/public/zfs/etc-zfs/zpool.d/r_ucor new file mode 100755 index 0000000..8ad3e10 --- /dev/null +++ b/straper/db/public/zfs/etc-zfs/zpool.d/r_ucor @@ -0,0 +1,249 @@ +#!/bin/sh +# +# Show SMART stats +# + +helpstr=" +smart: Show SMART temperature and error stats (specific to drive type) +smartx: Show SMART extended drive stats (specific to drive type). +temp: Show SMART drive temperature in celsius (all drives). +health: Show reported SMART status (all drives). +r_proc: Show SMART read GBytes processed over drive lifetime (SAS). +w_proc: Show SMART write GBytes processed over drive lifetime (SAS). +r_ucor: Show SMART read uncorrectable errors (SAS). +w_ucor: Show SMART write uncorrectable errors (SAS). +nonmed: Show SMART non-medium errors (SAS). +defect: Show SMART grown defect list (SAS). +hours_on: Show number of hours drive powered on (all drives). +realloc: Show SMART reallocated sectors count (ATA). +rep_ucor: Show SMART reported uncorrectable count (ATA). +cmd_to: Show SMART command timeout count (ATA). +pend_sec: Show SMART current pending sector count (ATA). +off_ucor: Show SMART offline uncorrectable errors (ATA). +ata_err: Show SMART ATA errors (ATA). +pwr_cyc: Show SMART power cycle count (ATA). +serial: Show disk serial number. +nvme_err: Show SMART NVMe errors (NVMe). +smart_test: Show SMART self-test results summary. +test_type: Show SMART self-test type (short, long... ). +test_status: Show SMART self-test status. +test_progress: Show SMART self-test percentage done. +test_ended: Show when the last SMART self-test ended (if supported). +" + +# Hack for developer testing +# +# If you set $samples to a directory containing smartctl output text files, +# we will use them instead of running smartctl on the vdevs. This can be +# useful if you want to test a bunch of different smartctl outputs. Also, if +# $samples is set, and additional 'file' column is added to the zpool output +# showing the filename. +samples= + +# get_filename_from_dir DIR +# +# Look in directory DIR and return a filename from it. The filename returned +# is chosen quasi-sequentially (based off our PID). This allows us to return +# a different filename every time this script is invoked (which we do for each +# vdev), without having to maintain state. +get_filename_from_dir() +{ + dir=$1 + pid="$$" + num_files=$(find "$dir" -maxdepth 1 -type f | wc -l) + mod=$((pid % num_files)) + i=0 + find "$dir" -type f -printf '%f\n' | while read -r file ; do + if [ "$mod" = "$i" ] ; then + echo "$file" + break + fi + i=$((i+1)) + done +} + +script="${0##*/}" + +if [ "$1" = "-h" ] ; then + echo "$helpstr" | grep "$script:" | tr -s '\t' | cut -f 2- + exit +fi + +# Sometimes, UPATH ends up /dev/(null). +# That should be corrected, but for now... +# shellcheck disable=SC2154 +if [ ! -b "$VDEV_UPATH" ]; then + somepath="${VDEV_PATH}" +else + somepath="${VDEV_UPATH}" +fi + +if [ -b "$somepath" ] && PATH="/usr/sbin:$PATH" command -v smartctl > /dev/null || [ -n "$samples" ] ; then + if [ -n "$samples" ] ; then + # cat a smartctl output text file instead of running smartctl + # on a vdev (only used for developer testing). + file=$(get_filename_from_dir "$samples") + echo "file=$file" + raw_out=$(cat "$samples/$file") + else + raw_out=$(sudo smartctl -a "$somepath") + fi + + # What kind of drive are we? Look for the right line in smartctl: + # + # SAS: + # Transport protocol: SAS + # + # SATA: + # ATA Version is: 8 + # + # NVMe: + # SMART/Health Information (NVMe Log 0xnn, NSID 0xnn) + # + out=$(echo "$raw_out" | awk ' +# SAS specific +/read:/{print "rrd="$4"\nr_cor="$5"\nr_proc="$7"\nr_ucor="$8} +/write:/{print "rwr="$4"\nw_cor="$5"\nw_proc="$7"\nw_ucor="$8} +/Non-medium error count/{print "nonmed="$4} +/Elements in grown defect list/{print "defect="$6} + +# SAS common +/SAS/{type="sas"} +/Drive Temperature:/{print "temp="$4} +# Status can be a long string, substitute spaces for '_' +/SMART Health Status:/{printf "health="; for(i=4;i<=NF-1;i++){printf "%s_", $i}; printf "%s\n", $i} +/number of hours powered up/{print "hours_on="$7; hours_on=int($7)} +/Serial number:/{print "serial="$3} + +# SATA specific +/Reallocated_Sector_Ct/{print "realloc="$10} +/Reported_Uncorrect/{print "rep_ucor="$10} +/Command_Timeout/{print "cmd_to="$10} +/Current_Pending_Sector/{print "pend_sec="$10} +/Offline_Uncorrectable/{print "off_ucor="$10} +/ATA Error Count:/{print "ata_err="$4} +/Power_Cycle_Count/{print "pwr_cyc="$10} + +# SATA common +/SATA/{type="sata"} +/Temperature_Celsius/{print "temp="$10} +/Airflow_Temperature_Cel/{print "temp="$10} +/Current Temperature:/{print "temp="$3} +/SMART overall-health self-assessment test result:/{print "health="$6} +/Power_On_Hours/{print "hours_on="$10; hours_on=int($10)} +/Serial Number:/{print "serial="$3} + +# NVMe common +/NVMe/{type="nvme"} +/Temperature:/{print "temp="$2} +/SMART overall-health self-assessment test result:/{print "health="$6} +/Power On Hours:/{gsub("[^0-9]","",$4); print "hours_on="$4} +/Serial Number:/{print "serial="$3} +/Power Cycles:/{print "pwr_cyc="$3} + +# NVMe specific +/Media and Data Integrity Errors:/{print "nvme_err="$6} + +# SMART self-test info +/Self-test execution status:/{progress=tolower($4)} # SAS +/SMART Self-test log/{test_seen=1} # SAS +/SMART Extended Self-test Log/{test_seen=1} # SATA +/# 1/{ + test_type=tolower($3"_"$4); + # Status could be one word ("Completed") or multiple ("Completed: read + # failure"). Look for the ":" to see if we need to grab more words. + + if ($5 ~ ":") + status=tolower($5""$6"_"$7) + else + status=tolower($5) + if (status=="self") + status="running"; + + if (type == "sas") { + hours=int($(NF-4)) + } else { + hours=int($(NF-1)) + # SATA reports percent remaining, rather than percent done + # Convert it to percent done. + progress=(100-int($(NF-2)))"%" + } + # When we int()-ify "hours", it converts stuff like "NOW" and "-" into + # 0. In those cases, set it to hours_on, so they will cancel out in + # the "hours_ago" calculation later on. + if (hours == 0) + hours=hours_on + + if (test_seen) { + print "test="hours_on + print "test_type="test_type + print "test_status="status + print "test_progress="progress + } + # Not all drives report hours_on + if (hours_on && hours) { + total_hours_ago=(hours_on-hours) + days_ago=int(total_hours_ago/24) + hours_ago=(total_hours_ago % 24) + if (days_ago != 0) + ago_str=days_ago"d" + if (hours_ago !=0) + ago_str=ago_str""hours_ago"h" + print "test_ended="ago_str + } +} + +END {print "type="type; ORS="\n"; print ""} +'); +fi +type=$(echo "$out" | grep '^type=' | cut -d '=' -f 2) + +# If type is not set by now, either we don't have a block device +# or smartctl failed. Either way, default to ATA and set $out to +# nothing. +if [ -z "$type" ]; then + type="sata" + out= +fi + +case $script in +smart) + # Print temperature plus common predictors of drive failure + if [ "$type" = "sas" ] ; then + scripts="temp|health|r_ucor|w_ucor" + elif [ "$type" = "sata" ] ; then + scripts="temp|health|ata_err|realloc|rep_ucor|cmd_to|pend_sec|off_ucor" + elif [ "$type" = "nvme" ] ; then + scripts="temp|health|nvme_err" + fi + ;; +smartx) + # Print some other interesting stats + if [ "$type" = "sas" ] ; then + scripts="hours_on|defect|nonmed|r_proc|w_proc" + elif [ "$type" = "sata" ] ; then + scripts="hours_on|pwr_cyc" + elif [ "$type" = "nvme" ] ; then + scripts="hours_on|pwr_cyc" + fi + ;; +smart_test) + scripts="test_type|test_status|test_progress|test_ended" + ;; +*) + scripts="$script" +esac + +with_vals=$(echo "$out" | grep -E "$scripts") +if [ -n "$with_vals" ]; then + echo "$with_vals" + without_vals=$(echo "$scripts" | tr '|' '\n' | + grep -v -E "$(echo "$with_vals" | + awk -F "=" '{print $1}')" | awk '{print $0"="}') +else + without_vals=$(echo "$scripts" | tr '|' '\n' | awk '{print $0"="}') +fi + +if [ -n "$without_vals" ]; then + echo "$without_vals" +fi diff --git a/straper/db/public/zfs/etc-zfs/zpool.d/realloc b/straper/db/public/zfs/etc-zfs/zpool.d/realloc new file mode 100755 index 0000000..8ad3e10 --- /dev/null +++ b/straper/db/public/zfs/etc-zfs/zpool.d/realloc @@ -0,0 +1,249 @@ +#!/bin/sh +# +# Show SMART stats +# + +helpstr=" +smart: Show SMART temperature and error stats (specific to drive type) +smartx: Show SMART extended drive stats (specific to drive type). +temp: Show SMART drive temperature in celsius (all drives). +health: Show reported SMART status (all drives). +r_proc: Show SMART read GBytes processed over drive lifetime (SAS). +w_proc: Show SMART write GBytes processed over drive lifetime (SAS). +r_ucor: Show SMART read uncorrectable errors (SAS). +w_ucor: Show SMART write uncorrectable errors (SAS). +nonmed: Show SMART non-medium errors (SAS). +defect: Show SMART grown defect list (SAS). +hours_on: Show number of hours drive powered on (all drives). +realloc: Show SMART reallocated sectors count (ATA). +rep_ucor: Show SMART reported uncorrectable count (ATA). +cmd_to: Show SMART command timeout count (ATA). +pend_sec: Show SMART current pending sector count (ATA). +off_ucor: Show SMART offline uncorrectable errors (ATA). +ata_err: Show SMART ATA errors (ATA). +pwr_cyc: Show SMART power cycle count (ATA). +serial: Show disk serial number. +nvme_err: Show SMART NVMe errors (NVMe). +smart_test: Show SMART self-test results summary. +test_type: Show SMART self-test type (short, long... ). +test_status: Show SMART self-test status. +test_progress: Show SMART self-test percentage done. +test_ended: Show when the last SMART self-test ended (if supported). +" + +# Hack for developer testing +# +# If you set $samples to a directory containing smartctl output text files, +# we will use them instead of running smartctl on the vdevs. This can be +# useful if you want to test a bunch of different smartctl outputs. Also, if +# $samples is set, and additional 'file' column is added to the zpool output +# showing the filename. +samples= + +# get_filename_from_dir DIR +# +# Look in directory DIR and return a filename from it. The filename returned +# is chosen quasi-sequentially (based off our PID). This allows us to return +# a different filename every time this script is invoked (which we do for each +# vdev), without having to maintain state. +get_filename_from_dir() +{ + dir=$1 + pid="$$" + num_files=$(find "$dir" -maxdepth 1 -type f | wc -l) + mod=$((pid % num_files)) + i=0 + find "$dir" -type f -printf '%f\n' | while read -r file ; do + if [ "$mod" = "$i" ] ; then + echo "$file" + break + fi + i=$((i+1)) + done +} + +script="${0##*/}" + +if [ "$1" = "-h" ] ; then + echo "$helpstr" | grep "$script:" | tr -s '\t' | cut -f 2- + exit +fi + +# Sometimes, UPATH ends up /dev/(null). +# That should be corrected, but for now... +# shellcheck disable=SC2154 +if [ ! -b "$VDEV_UPATH" ]; then + somepath="${VDEV_PATH}" +else + somepath="${VDEV_UPATH}" +fi + +if [ -b "$somepath" ] && PATH="/usr/sbin:$PATH" command -v smartctl > /dev/null || [ -n "$samples" ] ; then + if [ -n "$samples" ] ; then + # cat a smartctl output text file instead of running smartctl + # on a vdev (only used for developer testing). + file=$(get_filename_from_dir "$samples") + echo "file=$file" + raw_out=$(cat "$samples/$file") + else + raw_out=$(sudo smartctl -a "$somepath") + fi + + # What kind of drive are we? Look for the right line in smartctl: + # + # SAS: + # Transport protocol: SAS + # + # SATA: + # ATA Version is: 8 + # + # NVMe: + # SMART/Health Information (NVMe Log 0xnn, NSID 0xnn) + # + out=$(echo "$raw_out" | awk ' +# SAS specific +/read:/{print "rrd="$4"\nr_cor="$5"\nr_proc="$7"\nr_ucor="$8} +/write:/{print "rwr="$4"\nw_cor="$5"\nw_proc="$7"\nw_ucor="$8} +/Non-medium error count/{print "nonmed="$4} +/Elements in grown defect list/{print "defect="$6} + +# SAS common +/SAS/{type="sas"} +/Drive Temperature:/{print "temp="$4} +# Status can be a long string, substitute spaces for '_' +/SMART Health Status:/{printf "health="; for(i=4;i<=NF-1;i++){printf "%s_", $i}; printf "%s\n", $i} +/number of hours powered up/{print "hours_on="$7; hours_on=int($7)} +/Serial number:/{print "serial="$3} + +# SATA specific +/Reallocated_Sector_Ct/{print "realloc="$10} +/Reported_Uncorrect/{print "rep_ucor="$10} +/Command_Timeout/{print "cmd_to="$10} +/Current_Pending_Sector/{print "pend_sec="$10} +/Offline_Uncorrectable/{print "off_ucor="$10} +/ATA Error Count:/{print "ata_err="$4} +/Power_Cycle_Count/{print "pwr_cyc="$10} + +# SATA common +/SATA/{type="sata"} +/Temperature_Celsius/{print "temp="$10} +/Airflow_Temperature_Cel/{print "temp="$10} +/Current Temperature:/{print "temp="$3} +/SMART overall-health self-assessment test result:/{print "health="$6} +/Power_On_Hours/{print "hours_on="$10; hours_on=int($10)} +/Serial Number:/{print "serial="$3} + +# NVMe common +/NVMe/{type="nvme"} +/Temperature:/{print "temp="$2} +/SMART overall-health self-assessment test result:/{print "health="$6} +/Power On Hours:/{gsub("[^0-9]","",$4); print "hours_on="$4} +/Serial Number:/{print "serial="$3} +/Power Cycles:/{print "pwr_cyc="$3} + +# NVMe specific +/Media and Data Integrity Errors:/{print "nvme_err="$6} + +# SMART self-test info +/Self-test execution status:/{progress=tolower($4)} # SAS +/SMART Self-test log/{test_seen=1} # SAS +/SMART Extended Self-test Log/{test_seen=1} # SATA +/# 1/{ + test_type=tolower($3"_"$4); + # Status could be one word ("Completed") or multiple ("Completed: read + # failure"). Look for the ":" to see if we need to grab more words. + + if ($5 ~ ":") + status=tolower($5""$6"_"$7) + else + status=tolower($5) + if (status=="self") + status="running"; + + if (type == "sas") { + hours=int($(NF-4)) + } else { + hours=int($(NF-1)) + # SATA reports percent remaining, rather than percent done + # Convert it to percent done. + progress=(100-int($(NF-2)))"%" + } + # When we int()-ify "hours", it converts stuff like "NOW" and "-" into + # 0. In those cases, set it to hours_on, so they will cancel out in + # the "hours_ago" calculation later on. + if (hours == 0) + hours=hours_on + + if (test_seen) { + print "test="hours_on + print "test_type="test_type + print "test_status="status + print "test_progress="progress + } + # Not all drives report hours_on + if (hours_on && hours) { + total_hours_ago=(hours_on-hours) + days_ago=int(total_hours_ago/24) + hours_ago=(total_hours_ago % 24) + if (days_ago != 0) + ago_str=days_ago"d" + if (hours_ago !=0) + ago_str=ago_str""hours_ago"h" + print "test_ended="ago_str + } +} + +END {print "type="type; ORS="\n"; print ""} +'); +fi +type=$(echo "$out" | grep '^type=' | cut -d '=' -f 2) + +# If type is not set by now, either we don't have a block device +# or smartctl failed. Either way, default to ATA and set $out to +# nothing. +if [ -z "$type" ]; then + type="sata" + out= +fi + +case $script in +smart) + # Print temperature plus common predictors of drive failure + if [ "$type" = "sas" ] ; then + scripts="temp|health|r_ucor|w_ucor" + elif [ "$type" = "sata" ] ; then + scripts="temp|health|ata_err|realloc|rep_ucor|cmd_to|pend_sec|off_ucor" + elif [ "$type" = "nvme" ] ; then + scripts="temp|health|nvme_err" + fi + ;; +smartx) + # Print some other interesting stats + if [ "$type" = "sas" ] ; then + scripts="hours_on|defect|nonmed|r_proc|w_proc" + elif [ "$type" = "sata" ] ; then + scripts="hours_on|pwr_cyc" + elif [ "$type" = "nvme" ] ; then + scripts="hours_on|pwr_cyc" + fi + ;; +smart_test) + scripts="test_type|test_status|test_progress|test_ended" + ;; +*) + scripts="$script" +esac + +with_vals=$(echo "$out" | grep -E "$scripts") +if [ -n "$with_vals" ]; then + echo "$with_vals" + without_vals=$(echo "$scripts" | tr '|' '\n' | + grep -v -E "$(echo "$with_vals" | + awk -F "=" '{print $1}')" | awk '{print $0"="}') +else + without_vals=$(echo "$scripts" | tr '|' '\n' | awk '{print $0"="}') +fi + +if [ -n "$without_vals" ]; then + echo "$without_vals" +fi diff --git a/straper/db/public/zfs/etc-zfs/zpool.d/rep_ucor b/straper/db/public/zfs/etc-zfs/zpool.d/rep_ucor new file mode 100755 index 0000000..8ad3e10 --- /dev/null +++ b/straper/db/public/zfs/etc-zfs/zpool.d/rep_ucor @@ -0,0 +1,249 @@ +#!/bin/sh +# +# Show SMART stats +# + +helpstr=" +smart: Show SMART temperature and error stats (specific to drive type) +smartx: Show SMART extended drive stats (specific to drive type). +temp: Show SMART drive temperature in celsius (all drives). +health: Show reported SMART status (all drives). +r_proc: Show SMART read GBytes processed over drive lifetime (SAS). +w_proc: Show SMART write GBytes processed over drive lifetime (SAS). +r_ucor: Show SMART read uncorrectable errors (SAS). +w_ucor: Show SMART write uncorrectable errors (SAS). +nonmed: Show SMART non-medium errors (SAS). +defect: Show SMART grown defect list (SAS). +hours_on: Show number of hours drive powered on (all drives). +realloc: Show SMART reallocated sectors count (ATA). +rep_ucor: Show SMART reported uncorrectable count (ATA). +cmd_to: Show SMART command timeout count (ATA). +pend_sec: Show SMART current pending sector count (ATA). +off_ucor: Show SMART offline uncorrectable errors (ATA). +ata_err: Show SMART ATA errors (ATA). +pwr_cyc: Show SMART power cycle count (ATA). +serial: Show disk serial number. +nvme_err: Show SMART NVMe errors (NVMe). +smart_test: Show SMART self-test results summary. +test_type: Show SMART self-test type (short, long... ). +test_status: Show SMART self-test status. +test_progress: Show SMART self-test percentage done. +test_ended: Show when the last SMART self-test ended (if supported). +" + +# Hack for developer testing +# +# If you set $samples to a directory containing smartctl output text files, +# we will use them instead of running smartctl on the vdevs. This can be +# useful if you want to test a bunch of different smartctl outputs. Also, if +# $samples is set, and additional 'file' column is added to the zpool output +# showing the filename. +samples= + +# get_filename_from_dir DIR +# +# Look in directory DIR and return a filename from it. The filename returned +# is chosen quasi-sequentially (based off our PID). This allows us to return +# a different filename every time this script is invoked (which we do for each +# vdev), without having to maintain state. +get_filename_from_dir() +{ + dir=$1 + pid="$$" + num_files=$(find "$dir" -maxdepth 1 -type f | wc -l) + mod=$((pid % num_files)) + i=0 + find "$dir" -type f -printf '%f\n' | while read -r file ; do + if [ "$mod" = "$i" ] ; then + echo "$file" + break + fi + i=$((i+1)) + done +} + +script="${0##*/}" + +if [ "$1" = "-h" ] ; then + echo "$helpstr" | grep "$script:" | tr -s '\t' | cut -f 2- + exit +fi + +# Sometimes, UPATH ends up /dev/(null). +# That should be corrected, but for now... +# shellcheck disable=SC2154 +if [ ! -b "$VDEV_UPATH" ]; then + somepath="${VDEV_PATH}" +else + somepath="${VDEV_UPATH}" +fi + +if [ -b "$somepath" ] && PATH="/usr/sbin:$PATH" command -v smartctl > /dev/null || [ -n "$samples" ] ; then + if [ -n "$samples" ] ; then + # cat a smartctl output text file instead of running smartctl + # on a vdev (only used for developer testing). + file=$(get_filename_from_dir "$samples") + echo "file=$file" + raw_out=$(cat "$samples/$file") + else + raw_out=$(sudo smartctl -a "$somepath") + fi + + # What kind of drive are we? Look for the right line in smartctl: + # + # SAS: + # Transport protocol: SAS + # + # SATA: + # ATA Version is: 8 + # + # NVMe: + # SMART/Health Information (NVMe Log 0xnn, NSID 0xnn) + # + out=$(echo "$raw_out" | awk ' +# SAS specific +/read:/{print "rrd="$4"\nr_cor="$5"\nr_proc="$7"\nr_ucor="$8} +/write:/{print "rwr="$4"\nw_cor="$5"\nw_proc="$7"\nw_ucor="$8} +/Non-medium error count/{print "nonmed="$4} +/Elements in grown defect list/{print "defect="$6} + +# SAS common +/SAS/{type="sas"} +/Drive Temperature:/{print "temp="$4} +# Status can be a long string, substitute spaces for '_' +/SMART Health Status:/{printf "health="; for(i=4;i<=NF-1;i++){printf "%s_", $i}; printf "%s\n", $i} +/number of hours powered up/{print "hours_on="$7; hours_on=int($7)} +/Serial number:/{print "serial="$3} + +# SATA specific +/Reallocated_Sector_Ct/{print "realloc="$10} +/Reported_Uncorrect/{print "rep_ucor="$10} +/Command_Timeout/{print "cmd_to="$10} +/Current_Pending_Sector/{print "pend_sec="$10} +/Offline_Uncorrectable/{print "off_ucor="$10} +/ATA Error Count:/{print "ata_err="$4} +/Power_Cycle_Count/{print "pwr_cyc="$10} + +# SATA common +/SATA/{type="sata"} +/Temperature_Celsius/{print "temp="$10} +/Airflow_Temperature_Cel/{print "temp="$10} +/Current Temperature:/{print "temp="$3} +/SMART overall-health self-assessment test result:/{print "health="$6} +/Power_On_Hours/{print "hours_on="$10; hours_on=int($10)} +/Serial Number:/{print "serial="$3} + +# NVMe common +/NVMe/{type="nvme"} +/Temperature:/{print "temp="$2} +/SMART overall-health self-assessment test result:/{print "health="$6} +/Power On Hours:/{gsub("[^0-9]","",$4); print "hours_on="$4} +/Serial Number:/{print "serial="$3} +/Power Cycles:/{print "pwr_cyc="$3} + +# NVMe specific +/Media and Data Integrity Errors:/{print "nvme_err="$6} + +# SMART self-test info +/Self-test execution status:/{progress=tolower($4)} # SAS +/SMART Self-test log/{test_seen=1} # SAS +/SMART Extended Self-test Log/{test_seen=1} # SATA +/# 1/{ + test_type=tolower($3"_"$4); + # Status could be one word ("Completed") or multiple ("Completed: read + # failure"). Look for the ":" to see if we need to grab more words. + + if ($5 ~ ":") + status=tolower($5""$6"_"$7) + else + status=tolower($5) + if (status=="self") + status="running"; + + if (type == "sas") { + hours=int($(NF-4)) + } else { + hours=int($(NF-1)) + # SATA reports percent remaining, rather than percent done + # Convert it to percent done. + progress=(100-int($(NF-2)))"%" + } + # When we int()-ify "hours", it converts stuff like "NOW" and "-" into + # 0. In those cases, set it to hours_on, so they will cancel out in + # the "hours_ago" calculation later on. + if (hours == 0) + hours=hours_on + + if (test_seen) { + print "test="hours_on + print "test_type="test_type + print "test_status="status + print "test_progress="progress + } + # Not all drives report hours_on + if (hours_on && hours) { + total_hours_ago=(hours_on-hours) + days_ago=int(total_hours_ago/24) + hours_ago=(total_hours_ago % 24) + if (days_ago != 0) + ago_str=days_ago"d" + if (hours_ago !=0) + ago_str=ago_str""hours_ago"h" + print "test_ended="ago_str + } +} + +END {print "type="type; ORS="\n"; print ""} +'); +fi +type=$(echo "$out" | grep '^type=' | cut -d '=' -f 2) + +# If type is not set by now, either we don't have a block device +# or smartctl failed. Either way, default to ATA and set $out to +# nothing. +if [ -z "$type" ]; then + type="sata" + out= +fi + +case $script in +smart) + # Print temperature plus common predictors of drive failure + if [ "$type" = "sas" ] ; then + scripts="temp|health|r_ucor|w_ucor" + elif [ "$type" = "sata" ] ; then + scripts="temp|health|ata_err|realloc|rep_ucor|cmd_to|pend_sec|off_ucor" + elif [ "$type" = "nvme" ] ; then + scripts="temp|health|nvme_err" + fi + ;; +smartx) + # Print some other interesting stats + if [ "$type" = "sas" ] ; then + scripts="hours_on|defect|nonmed|r_proc|w_proc" + elif [ "$type" = "sata" ] ; then + scripts="hours_on|pwr_cyc" + elif [ "$type" = "nvme" ] ; then + scripts="hours_on|pwr_cyc" + fi + ;; +smart_test) + scripts="test_type|test_status|test_progress|test_ended" + ;; +*) + scripts="$script" +esac + +with_vals=$(echo "$out" | grep -E "$scripts") +if [ -n "$with_vals" ]; then + echo "$with_vals" + without_vals=$(echo "$scripts" | tr '|' '\n' | + grep -v -E "$(echo "$with_vals" | + awk -F "=" '{print $1}')" | awk '{print $0"="}') +else + without_vals=$(echo "$scripts" | tr '|' '\n' | awk '{print $0"="}') +fi + +if [ -n "$without_vals" ]; then + echo "$without_vals" +fi diff --git a/straper/db/public/zfs/etc-zfs/zpool.d/serial b/straper/db/public/zfs/etc-zfs/zpool.d/serial new file mode 100755 index 0000000..8ad3e10 --- /dev/null +++ b/straper/db/public/zfs/etc-zfs/zpool.d/serial @@ -0,0 +1,249 @@ +#!/bin/sh +# +# Show SMART stats +# + +helpstr=" +smart: Show SMART temperature and error stats (specific to drive type) +smartx: Show SMART extended drive stats (specific to drive type). +temp: Show SMART drive temperature in celsius (all drives). +health: Show reported SMART status (all drives). +r_proc: Show SMART read GBytes processed over drive lifetime (SAS). +w_proc: Show SMART write GBytes processed over drive lifetime (SAS). +r_ucor: Show SMART read uncorrectable errors (SAS). +w_ucor: Show SMART write uncorrectable errors (SAS). +nonmed: Show SMART non-medium errors (SAS). +defect: Show SMART grown defect list (SAS). +hours_on: Show number of hours drive powered on (all drives). +realloc: Show SMART reallocated sectors count (ATA). +rep_ucor: Show SMART reported uncorrectable count (ATA). +cmd_to: Show SMART command timeout count (ATA). +pend_sec: Show SMART current pending sector count (ATA). +off_ucor: Show SMART offline uncorrectable errors (ATA). +ata_err: Show SMART ATA errors (ATA). +pwr_cyc: Show SMART power cycle count (ATA). +serial: Show disk serial number. +nvme_err: Show SMART NVMe errors (NVMe). +smart_test: Show SMART self-test results summary. +test_type: Show SMART self-test type (short, long... ). +test_status: Show SMART self-test status. +test_progress: Show SMART self-test percentage done. +test_ended: Show when the last SMART self-test ended (if supported). +" + +# Hack for developer testing +# +# If you set $samples to a directory containing smartctl output text files, +# we will use them instead of running smartctl on the vdevs. This can be +# useful if you want to test a bunch of different smartctl outputs. Also, if +# $samples is set, and additional 'file' column is added to the zpool output +# showing the filename. +samples= + +# get_filename_from_dir DIR +# +# Look in directory DIR and return a filename from it. The filename returned +# is chosen quasi-sequentially (based off our PID). This allows us to return +# a different filename every time this script is invoked (which we do for each +# vdev), without having to maintain state. +get_filename_from_dir() +{ + dir=$1 + pid="$$" + num_files=$(find "$dir" -maxdepth 1 -type f | wc -l) + mod=$((pid % num_files)) + i=0 + find "$dir" -type f -printf '%f\n' | while read -r file ; do + if [ "$mod" = "$i" ] ; then + echo "$file" + break + fi + i=$((i+1)) + done +} + +script="${0##*/}" + +if [ "$1" = "-h" ] ; then + echo "$helpstr" | grep "$script:" | tr -s '\t' | cut -f 2- + exit +fi + +# Sometimes, UPATH ends up /dev/(null). +# That should be corrected, but for now... +# shellcheck disable=SC2154 +if [ ! -b "$VDEV_UPATH" ]; then + somepath="${VDEV_PATH}" +else + somepath="${VDEV_UPATH}" +fi + +if [ -b "$somepath" ] && PATH="/usr/sbin:$PATH" command -v smartctl > /dev/null || [ -n "$samples" ] ; then + if [ -n "$samples" ] ; then + # cat a smartctl output text file instead of running smartctl + # on a vdev (only used for developer testing). + file=$(get_filename_from_dir "$samples") + echo "file=$file" + raw_out=$(cat "$samples/$file") + else + raw_out=$(sudo smartctl -a "$somepath") + fi + + # What kind of drive are we? Look for the right line in smartctl: + # + # SAS: + # Transport protocol: SAS + # + # SATA: + # ATA Version is: 8 + # + # NVMe: + # SMART/Health Information (NVMe Log 0xnn, NSID 0xnn) + # + out=$(echo "$raw_out" | awk ' +# SAS specific +/read:/{print "rrd="$4"\nr_cor="$5"\nr_proc="$7"\nr_ucor="$8} +/write:/{print "rwr="$4"\nw_cor="$5"\nw_proc="$7"\nw_ucor="$8} +/Non-medium error count/{print "nonmed="$4} +/Elements in grown defect list/{print "defect="$6} + +# SAS common +/SAS/{type="sas"} +/Drive Temperature:/{print "temp="$4} +# Status can be a long string, substitute spaces for '_' +/SMART Health Status:/{printf "health="; for(i=4;i<=NF-1;i++){printf "%s_", $i}; printf "%s\n", $i} +/number of hours powered up/{print "hours_on="$7; hours_on=int($7)} +/Serial number:/{print "serial="$3} + +# SATA specific +/Reallocated_Sector_Ct/{print "realloc="$10} +/Reported_Uncorrect/{print "rep_ucor="$10} +/Command_Timeout/{print "cmd_to="$10} +/Current_Pending_Sector/{print "pend_sec="$10} +/Offline_Uncorrectable/{print "off_ucor="$10} +/ATA Error Count:/{print "ata_err="$4} +/Power_Cycle_Count/{print "pwr_cyc="$10} + +# SATA common +/SATA/{type="sata"} +/Temperature_Celsius/{print "temp="$10} +/Airflow_Temperature_Cel/{print "temp="$10} +/Current Temperature:/{print "temp="$3} +/SMART overall-health self-assessment test result:/{print "health="$6} +/Power_On_Hours/{print "hours_on="$10; hours_on=int($10)} +/Serial Number:/{print "serial="$3} + +# NVMe common +/NVMe/{type="nvme"} +/Temperature:/{print "temp="$2} +/SMART overall-health self-assessment test result:/{print "health="$6} +/Power On Hours:/{gsub("[^0-9]","",$4); print "hours_on="$4} +/Serial Number:/{print "serial="$3} +/Power Cycles:/{print "pwr_cyc="$3} + +# NVMe specific +/Media and Data Integrity Errors:/{print "nvme_err="$6} + +# SMART self-test info +/Self-test execution status:/{progress=tolower($4)} # SAS +/SMART Self-test log/{test_seen=1} # SAS +/SMART Extended Self-test Log/{test_seen=1} # SATA +/# 1/{ + test_type=tolower($3"_"$4); + # Status could be one word ("Completed") or multiple ("Completed: read + # failure"). Look for the ":" to see if we need to grab more words. + + if ($5 ~ ":") + status=tolower($5""$6"_"$7) + else + status=tolower($5) + if (status=="self") + status="running"; + + if (type == "sas") { + hours=int($(NF-4)) + } else { + hours=int($(NF-1)) + # SATA reports percent remaining, rather than percent done + # Convert it to percent done. + progress=(100-int($(NF-2)))"%" + } + # When we int()-ify "hours", it converts stuff like "NOW" and "-" into + # 0. In those cases, set it to hours_on, so they will cancel out in + # the "hours_ago" calculation later on. + if (hours == 0) + hours=hours_on + + if (test_seen) { + print "test="hours_on + print "test_type="test_type + print "test_status="status + print "test_progress="progress + } + # Not all drives report hours_on + if (hours_on && hours) { + total_hours_ago=(hours_on-hours) + days_ago=int(total_hours_ago/24) + hours_ago=(total_hours_ago % 24) + if (days_ago != 0) + ago_str=days_ago"d" + if (hours_ago !=0) + ago_str=ago_str""hours_ago"h" + print "test_ended="ago_str + } +} + +END {print "type="type; ORS="\n"; print ""} +'); +fi +type=$(echo "$out" | grep '^type=' | cut -d '=' -f 2) + +# If type is not set by now, either we don't have a block device +# or smartctl failed. Either way, default to ATA and set $out to +# nothing. +if [ -z "$type" ]; then + type="sata" + out= +fi + +case $script in +smart) + # Print temperature plus common predictors of drive failure + if [ "$type" = "sas" ] ; then + scripts="temp|health|r_ucor|w_ucor" + elif [ "$type" = "sata" ] ; then + scripts="temp|health|ata_err|realloc|rep_ucor|cmd_to|pend_sec|off_ucor" + elif [ "$type" = "nvme" ] ; then + scripts="temp|health|nvme_err" + fi + ;; +smartx) + # Print some other interesting stats + if [ "$type" = "sas" ] ; then + scripts="hours_on|defect|nonmed|r_proc|w_proc" + elif [ "$type" = "sata" ] ; then + scripts="hours_on|pwr_cyc" + elif [ "$type" = "nvme" ] ; then + scripts="hours_on|pwr_cyc" + fi + ;; +smart_test) + scripts="test_type|test_status|test_progress|test_ended" + ;; +*) + scripts="$script" +esac + +with_vals=$(echo "$out" | grep -E "$scripts") +if [ -n "$with_vals" ]; then + echo "$with_vals" + without_vals=$(echo "$scripts" | tr '|' '\n' | + grep -v -E "$(echo "$with_vals" | + awk -F "=" '{print $1}')" | awk '{print $0"="}') +else + without_vals=$(echo "$scripts" | tr '|' '\n' | awk '{print $0"="}') +fi + +if [ -n "$without_vals" ]; then + echo "$without_vals" +fi diff --git a/straper/db/public/zfs/etc-zfs/zpool.d/ses b/straper/db/public/zfs/etc-zfs/zpool.d/ses new file mode 100755 index 0000000..19ef92a --- /dev/null +++ b/straper/db/public/zfs/etc-zfs/zpool.d/ses @@ -0,0 +1,69 @@ +#!/bin/sh +# +# Print SCSI Enclosure Services (SES) info. The output is dependent on the name +# of the script/symlink used to call it. +# +helpstr=" +enc: Show disk enclosure w:x:y:z value. +slot: Show disk slot number as reported by the enclosure. +encdev: Show /dev/sg* device associated with the enclosure disk slot. +fault_led: Show value of the disk enclosure slot fault LED. +locate_led: Show value of the disk enclosure slot locate LED. +ses: Show disk's enc, enc device, slot, and fault/locate LED values." + +script="${0##*/}" +if [ "$1" = "-h" ] ; then + echo "$helpstr" | grep "$script:" | tr -s '\t' | cut -f 2- + exit +fi + +if [ "$script" = "ses" ] ; then + scripts='enc encdev slot fault_led locate_led' +else + scripts="$script" +fi + +for i in $scripts ; do + # shellcheck disable=SC2154 + if [ -z "$VDEV_ENC_SYSFS_PATH" ] ; then + echo "$i=" + continue + fi + + val="" + case $i in + enc) + if echo "$VDEV_ENC_SYSFS_PATH" | grep -q '/sys/bus/pci/slots' ; then + val="$VDEV_ENC_SYSFS_PATH" + else + val="$(ls """$VDEV_ENC_SYSFS_PATH/../../""" 2>/dev/null)" + fi + ;; + slot) + if echo "$VDEV_ENC_SYSFS_PATH" | grep -q '/sys/bus/pci/slots' ; then + val="$(basename """$VDEV_ENC_SYSFS_PATH""")" + else + val="$(cat """$VDEV_ENC_SYSFS_PATH/slot""" 2>/dev/null)" + fi + ;; + encdev) + val=$(ls "$VDEV_ENC_SYSFS_PATH/../device/scsi_generic" 2>/dev/null) + ;; + fault_led) + # JBODs fault LED is called 'fault', NVMe fault LED is called + # 'attention'. + if [ -f "$VDEV_ENC_SYSFS_PATH/fault" ] ; then + val=$(cat "$VDEV_ENC_SYSFS_PATH/fault" 2>/dev/null) + elif [ -f "$VDEV_ENC_SYSFS_PATH/attention" ] ; then + val=$(cat "$VDEV_ENC_SYSFS_PATH/attention" 2>/dev/null) + fi + ;; + locate_led) + val=$(cat "$VDEV_ENC_SYSFS_PATH/locate" 2>/dev/null) + ;; + *) + val=invalid + ;; + esac + echo "$i=$val" +done diff --git a/straper/db/public/zfs/etc-zfs/zpool.d/size b/straper/db/public/zfs/etc-zfs/zpool.d/size new file mode 100755 index 0000000..293effd --- /dev/null +++ b/straper/db/public/zfs/etc-zfs/zpool.d/size @@ -0,0 +1,92 @@ +#!/bin/sh +# +# Print some common lsblk values +# +# Any (lowercased) name symlinked to the lsblk script will be passed to lsblk +# as one of its --output names. Here's a partial list of --output names +# from the lsblk binary: +# +# Available columns (for --output): +# NAME device name +# KNAME internal kernel device name +# MAJ:MIN major:minor device number +# FSTYPE filesystem type +# MOUNTPOINT where the device is mounted +# LABEL filesystem LABEL +# UUID filesystem UUID +# RA read-ahead of the device +# RO read-only device +# RM removable device +# MODEL device identifier +# SIZE size of the device +# STATE state of the device +# OWNER user name +# GROUP group name +# MODE device node permissions +# ALIGNMENT alignment offset +# MIN-IO minimum I/O size +# OPT-IO optimal I/O size +# PHY-SEC physical sector size +# LOG-SEC logical sector size +# ROTA rotational device +# SCHED I/O scheduler name +# RQ-SIZE request queue size +# TYPE device type +# DISC-ALN discard alignment offset +# DISC-GRAN discard granularity +# DISC-MAX discard max bytes +# DISC-ZERO discard zeroes data +# +# If the script is run as just 'lsblk' then print out disk size, vendor, +# and model number. + + +helpstr=" +label: Show filesystem label. +model: Show disk model number. +size: Show the disk capacity. +vendor: Show the disk vendor. +lsblk: Show the disk size, vendor, and model number." + +script="${0##*/}" + +if [ "$1" = "-h" ] ; then + echo "$helpstr" | grep "$script:" | tr -s '\t' | cut -f 2- + exit +fi + +if [ "$script" = "lsblk" ] ; then + list="size vendor model" +else + list=$(echo "$script" | tr '[:upper:]' '[:lower:]') +fi + +# Sometimes, UPATH ends up /dev/(null). +# That should be corrected, but for now... +# shellcheck disable=SC2154 +if [ ! -b "$VDEV_UPATH" ]; then + somepath="${VDEV_PATH}" +else + somepath="${VDEV_UPATH}" +fi + +# Older versions of lsblk don't support all these values (like SERIAL). +for i in $list ; do + + # Special case: Looking up the size of a file-based vdev can't + # be done with lsblk. + if [ "$i" = "size" ] && [ -f "$somepath" ] ; then + size=$(du -h --apparent-size "$somepath" | cut -f 1) + echo "size=$size" + continue + fi + + + val="" + if val=$(eval "lsblk -dl -n -o $i $somepath 2>/dev/null") ; then + # Remove leading/trailing whitespace from value + val=$(echo "$val" | sed -e 's/^[[:space:]]*//' \ + -e 's/[[:space:]]*$//') + fi + echo "$i=$val" +done diff --git a/straper/db/public/zfs/etc-zfs/zpool.d/slot b/straper/db/public/zfs/etc-zfs/zpool.d/slot new file mode 100755 index 0000000..19ef92a --- /dev/null +++ b/straper/db/public/zfs/etc-zfs/zpool.d/slot @@ -0,0 +1,69 @@ +#!/bin/sh +# +# Print SCSI Enclosure Services (SES) info. The output is dependent on the name +# of the script/symlink used to call it. +# +helpstr=" +enc: Show disk enclosure w:x:y:z value. +slot: Show disk slot number as reported by the enclosure. +encdev: Show /dev/sg* device associated with the enclosure disk slot. +fault_led: Show value of the disk enclosure slot fault LED. +locate_led: Show value of the disk enclosure slot locate LED. +ses: Show disk's enc, enc device, slot, and fault/locate LED values." + +script="${0##*/}" +if [ "$1" = "-h" ] ; then + echo "$helpstr" | grep "$script:" | tr -s '\t' | cut -f 2- + exit +fi + +if [ "$script" = "ses" ] ; then + scripts='enc encdev slot fault_led locate_led' +else + scripts="$script" +fi + +for i in $scripts ; do + # shellcheck disable=SC2154 + if [ -z "$VDEV_ENC_SYSFS_PATH" ] ; then + echo "$i=" + continue + fi + + val="" + case $i in + enc) + if echo "$VDEV_ENC_SYSFS_PATH" | grep -q '/sys/bus/pci/slots' ; then + val="$VDEV_ENC_SYSFS_PATH" + else + val="$(ls """$VDEV_ENC_SYSFS_PATH/../../""" 2>/dev/null)" + fi + ;; + slot) + if echo "$VDEV_ENC_SYSFS_PATH" | grep -q '/sys/bus/pci/slots' ; then + val="$(basename """$VDEV_ENC_SYSFS_PATH""")" + else + val="$(cat """$VDEV_ENC_SYSFS_PATH/slot""" 2>/dev/null)" + fi + ;; + encdev) + val=$(ls "$VDEV_ENC_SYSFS_PATH/../device/scsi_generic" 2>/dev/null) + ;; + fault_led) + # JBODs fault LED is called 'fault', NVMe fault LED is called + # 'attention'. + if [ -f "$VDEV_ENC_SYSFS_PATH/fault" ] ; then + val=$(cat "$VDEV_ENC_SYSFS_PATH/fault" 2>/dev/null) + elif [ -f "$VDEV_ENC_SYSFS_PATH/attention" ] ; then + val=$(cat "$VDEV_ENC_SYSFS_PATH/attention" 2>/dev/null) + fi + ;; + locate_led) + val=$(cat "$VDEV_ENC_SYSFS_PATH/locate" 2>/dev/null) + ;; + *) + val=invalid + ;; + esac + echo "$i=$val" +done diff --git a/straper/db/public/zfs/etc-zfs/zpool.d/smart b/straper/db/public/zfs/etc-zfs/zpool.d/smart new file mode 100755 index 0000000..8ad3e10 --- /dev/null +++ b/straper/db/public/zfs/etc-zfs/zpool.d/smart @@ -0,0 +1,249 @@ +#!/bin/sh +# +# Show SMART stats +# + +helpstr=" +smart: Show SMART temperature and error stats (specific to drive type) +smartx: Show SMART extended drive stats (specific to drive type). +temp: Show SMART drive temperature in celsius (all drives). +health: Show reported SMART status (all drives). +r_proc: Show SMART read GBytes processed over drive lifetime (SAS). +w_proc: Show SMART write GBytes processed over drive lifetime (SAS). +r_ucor: Show SMART read uncorrectable errors (SAS). +w_ucor: Show SMART write uncorrectable errors (SAS). +nonmed: Show SMART non-medium errors (SAS). +defect: Show SMART grown defect list (SAS). +hours_on: Show number of hours drive powered on (all drives). +realloc: Show SMART reallocated sectors count (ATA). +rep_ucor: Show SMART reported uncorrectable count (ATA). +cmd_to: Show SMART command timeout count (ATA). +pend_sec: Show SMART current pending sector count (ATA). +off_ucor: Show SMART offline uncorrectable errors (ATA). +ata_err: Show SMART ATA errors (ATA). +pwr_cyc: Show SMART power cycle count (ATA). +serial: Show disk serial number. +nvme_err: Show SMART NVMe errors (NVMe). +smart_test: Show SMART self-test results summary. +test_type: Show SMART self-test type (short, long... ). +test_status: Show SMART self-test status. +test_progress: Show SMART self-test percentage done. +test_ended: Show when the last SMART self-test ended (if supported). +" + +# Hack for developer testing +# +# If you set $samples to a directory containing smartctl output text files, +# we will use them instead of running smartctl on the vdevs. This can be +# useful if you want to test a bunch of different smartctl outputs. Also, if +# $samples is set, and additional 'file' column is added to the zpool output +# showing the filename. +samples= + +# get_filename_from_dir DIR +# +# Look in directory DIR and return a filename from it. The filename returned +# is chosen quasi-sequentially (based off our PID). This allows us to return +# a different filename every time this script is invoked (which we do for each +# vdev), without having to maintain state. +get_filename_from_dir() +{ + dir=$1 + pid="$$" + num_files=$(find "$dir" -maxdepth 1 -type f | wc -l) + mod=$((pid % num_files)) + i=0 + find "$dir" -type f -printf '%f\n' | while read -r file ; do + if [ "$mod" = "$i" ] ; then + echo "$file" + break + fi + i=$((i+1)) + done +} + +script="${0##*/}" + +if [ "$1" = "-h" ] ; then + echo "$helpstr" | grep "$script:" | tr -s '\t' | cut -f 2- + exit +fi + +# Sometimes, UPATH ends up /dev/(null). +# That should be corrected, but for now... +# shellcheck disable=SC2154 +if [ ! -b "$VDEV_UPATH" ]; then + somepath="${VDEV_PATH}" +else + somepath="${VDEV_UPATH}" +fi + +if [ -b "$somepath" ] && PATH="/usr/sbin:$PATH" command -v smartctl > /dev/null || [ -n "$samples" ] ; then + if [ -n "$samples" ] ; then + # cat a smartctl output text file instead of running smartctl + # on a vdev (only used for developer testing). + file=$(get_filename_from_dir "$samples") + echo "file=$file" + raw_out=$(cat "$samples/$file") + else + raw_out=$(sudo smartctl -a "$somepath") + fi + + # What kind of drive are we? Look for the right line in smartctl: + # + # SAS: + # Transport protocol: SAS + # + # SATA: + # ATA Version is: 8 + # + # NVMe: + # SMART/Health Information (NVMe Log 0xnn, NSID 0xnn) + # + out=$(echo "$raw_out" | awk ' +# SAS specific +/read:/{print "rrd="$4"\nr_cor="$5"\nr_proc="$7"\nr_ucor="$8} +/write:/{print "rwr="$4"\nw_cor="$5"\nw_proc="$7"\nw_ucor="$8} +/Non-medium error count/{print "nonmed="$4} +/Elements in grown defect list/{print "defect="$6} + +# SAS common +/SAS/{type="sas"} +/Drive Temperature:/{print "temp="$4} +# Status can be a long string, substitute spaces for '_' +/SMART Health Status:/{printf "health="; for(i=4;i<=NF-1;i++){printf "%s_", $i}; printf "%s\n", $i} +/number of hours powered up/{print "hours_on="$7; hours_on=int($7)} +/Serial number:/{print "serial="$3} + +# SATA specific +/Reallocated_Sector_Ct/{print "realloc="$10} +/Reported_Uncorrect/{print "rep_ucor="$10} +/Command_Timeout/{print "cmd_to="$10} +/Current_Pending_Sector/{print "pend_sec="$10} +/Offline_Uncorrectable/{print "off_ucor="$10} +/ATA Error Count:/{print "ata_err="$4} +/Power_Cycle_Count/{print "pwr_cyc="$10} + +# SATA common +/SATA/{type="sata"} +/Temperature_Celsius/{print "temp="$10} +/Airflow_Temperature_Cel/{print "temp="$10} +/Current Temperature:/{print "temp="$3} +/SMART overall-health self-assessment test result:/{print "health="$6} +/Power_On_Hours/{print "hours_on="$10; hours_on=int($10)} +/Serial Number:/{print "serial="$3} + +# NVMe common +/NVMe/{type="nvme"} +/Temperature:/{print "temp="$2} +/SMART overall-health self-assessment test result:/{print "health="$6} +/Power On Hours:/{gsub("[^0-9]","",$4); print "hours_on="$4} +/Serial Number:/{print "serial="$3} +/Power Cycles:/{print "pwr_cyc="$3} + +# NVMe specific +/Media and Data Integrity Errors:/{print "nvme_err="$6} + +# SMART self-test info +/Self-test execution status:/{progress=tolower($4)} # SAS +/SMART Self-test log/{test_seen=1} # SAS +/SMART Extended Self-test Log/{test_seen=1} # SATA +/# 1/{ + test_type=tolower($3"_"$4); + # Status could be one word ("Completed") or multiple ("Completed: read + # failure"). Look for the ":" to see if we need to grab more words. + + if ($5 ~ ":") + status=tolower($5""$6"_"$7) + else + status=tolower($5) + if (status=="self") + status="running"; + + if (type == "sas") { + hours=int($(NF-4)) + } else { + hours=int($(NF-1)) + # SATA reports percent remaining, rather than percent done + # Convert it to percent done. + progress=(100-int($(NF-2)))"%" + } + # When we int()-ify "hours", it converts stuff like "NOW" and "-" into + # 0. In those cases, set it to hours_on, so they will cancel out in + # the "hours_ago" calculation later on. + if (hours == 0) + hours=hours_on + + if (test_seen) { + print "test="hours_on + print "test_type="test_type + print "test_status="status + print "test_progress="progress + } + # Not all drives report hours_on + if (hours_on && hours) { + total_hours_ago=(hours_on-hours) + days_ago=int(total_hours_ago/24) + hours_ago=(total_hours_ago % 24) + if (days_ago != 0) + ago_str=days_ago"d" + if (hours_ago !=0) + ago_str=ago_str""hours_ago"h" + print "test_ended="ago_str + } +} + +END {print "type="type; ORS="\n"; print ""} +'); +fi +type=$(echo "$out" | grep '^type=' | cut -d '=' -f 2) + +# If type is not set by now, either we don't have a block device +# or smartctl failed. Either way, default to ATA and set $out to +# nothing. +if [ -z "$type" ]; then + type="sata" + out= +fi + +case $script in +smart) + # Print temperature plus common predictors of drive failure + if [ "$type" = "sas" ] ; then + scripts="temp|health|r_ucor|w_ucor" + elif [ "$type" = "sata" ] ; then + scripts="temp|health|ata_err|realloc|rep_ucor|cmd_to|pend_sec|off_ucor" + elif [ "$type" = "nvme" ] ; then + scripts="temp|health|nvme_err" + fi + ;; +smartx) + # Print some other interesting stats + if [ "$type" = "sas" ] ; then + scripts="hours_on|defect|nonmed|r_proc|w_proc" + elif [ "$type" = "sata" ] ; then + scripts="hours_on|pwr_cyc" + elif [ "$type" = "nvme" ] ; then + scripts="hours_on|pwr_cyc" + fi + ;; +smart_test) + scripts="test_type|test_status|test_progress|test_ended" + ;; +*) + scripts="$script" +esac + +with_vals=$(echo "$out" | grep -E "$scripts") +if [ -n "$with_vals" ]; then + echo "$with_vals" + without_vals=$(echo "$scripts" | tr '|' '\n' | + grep -v -E "$(echo "$with_vals" | + awk -F "=" '{print $1}')" | awk '{print $0"="}') +else + without_vals=$(echo "$scripts" | tr '|' '\n' | awk '{print $0"="}') +fi + +if [ -n "$without_vals" ]; then + echo "$without_vals" +fi diff --git a/straper/db/public/zfs/etc-zfs/zpool.d/smart_test b/straper/db/public/zfs/etc-zfs/zpool.d/smart_test new file mode 100755 index 0000000..8ad3e10 --- /dev/null +++ b/straper/db/public/zfs/etc-zfs/zpool.d/smart_test @@ -0,0 +1,249 @@ +#!/bin/sh +# +# Show SMART stats +# + +helpstr=" +smart: Show SMART temperature and error stats (specific to drive type) +smartx: Show SMART extended drive stats (specific to drive type). +temp: Show SMART drive temperature in celsius (all drives). +health: Show reported SMART status (all drives). +r_proc: Show SMART read GBytes processed over drive lifetime (SAS). +w_proc: Show SMART write GBytes processed over drive lifetime (SAS). +r_ucor: Show SMART read uncorrectable errors (SAS). +w_ucor: Show SMART write uncorrectable errors (SAS). +nonmed: Show SMART non-medium errors (SAS). +defect: Show SMART grown defect list (SAS). +hours_on: Show number of hours drive powered on (all drives). +realloc: Show SMART reallocated sectors count (ATA). +rep_ucor: Show SMART reported uncorrectable count (ATA). +cmd_to: Show SMART command timeout count (ATA). +pend_sec: Show SMART current pending sector count (ATA). +off_ucor: Show SMART offline uncorrectable errors (ATA). +ata_err: Show SMART ATA errors (ATA). +pwr_cyc: Show SMART power cycle count (ATA). +serial: Show disk serial number. +nvme_err: Show SMART NVMe errors (NVMe). +smart_test: Show SMART self-test results summary. +test_type: Show SMART self-test type (short, long... ). +test_status: Show SMART self-test status. +test_progress: Show SMART self-test percentage done. +test_ended: Show when the last SMART self-test ended (if supported). +" + +# Hack for developer testing +# +# If you set $samples to a directory containing smartctl output text files, +# we will use them instead of running smartctl on the vdevs. This can be +# useful if you want to test a bunch of different smartctl outputs. Also, if +# $samples is set, and additional 'file' column is added to the zpool output +# showing the filename. +samples= + +# get_filename_from_dir DIR +# +# Look in directory DIR and return a filename from it. The filename returned +# is chosen quasi-sequentially (based off our PID). This allows us to return +# a different filename every time this script is invoked (which we do for each +# vdev), without having to maintain state. +get_filename_from_dir() +{ + dir=$1 + pid="$$" + num_files=$(find "$dir" -maxdepth 1 -type f | wc -l) + mod=$((pid % num_files)) + i=0 + find "$dir" -type f -printf '%f\n' | while read -r file ; do + if [ "$mod" = "$i" ] ; then + echo "$file" + break + fi + i=$((i+1)) + done +} + +script="${0##*/}" + +if [ "$1" = "-h" ] ; then + echo "$helpstr" | grep "$script:" | tr -s '\t' | cut -f 2- + exit +fi + +# Sometimes, UPATH ends up /dev/(null). +# That should be corrected, but for now... +# shellcheck disable=SC2154 +if [ ! -b "$VDEV_UPATH" ]; then + somepath="${VDEV_PATH}" +else + somepath="${VDEV_UPATH}" +fi + +if [ -b "$somepath" ] && PATH="/usr/sbin:$PATH" command -v smartctl > /dev/null || [ -n "$samples" ] ; then + if [ -n "$samples" ] ; then + # cat a smartctl output text file instead of running smartctl + # on a vdev (only used for developer testing). + file=$(get_filename_from_dir "$samples") + echo "file=$file" + raw_out=$(cat "$samples/$file") + else + raw_out=$(sudo smartctl -a "$somepath") + fi + + # What kind of drive are we? Look for the right line in smartctl: + # + # SAS: + # Transport protocol: SAS + # + # SATA: + # ATA Version is: 8 + # + # NVMe: + # SMART/Health Information (NVMe Log 0xnn, NSID 0xnn) + # + out=$(echo "$raw_out" | awk ' +# SAS specific +/read:/{print "rrd="$4"\nr_cor="$5"\nr_proc="$7"\nr_ucor="$8} +/write:/{print "rwr="$4"\nw_cor="$5"\nw_proc="$7"\nw_ucor="$8} +/Non-medium error count/{print "nonmed="$4} +/Elements in grown defect list/{print "defect="$6} + +# SAS common +/SAS/{type="sas"} +/Drive Temperature:/{print "temp="$4} +# Status can be a long string, substitute spaces for '_' +/SMART Health Status:/{printf "health="; for(i=4;i<=NF-1;i++){printf "%s_", $i}; printf "%s\n", $i} +/number of hours powered up/{print "hours_on="$7; hours_on=int($7)} +/Serial number:/{print "serial="$3} + +# SATA specific +/Reallocated_Sector_Ct/{print "realloc="$10} +/Reported_Uncorrect/{print "rep_ucor="$10} +/Command_Timeout/{print "cmd_to="$10} +/Current_Pending_Sector/{print "pend_sec="$10} +/Offline_Uncorrectable/{print "off_ucor="$10} +/ATA Error Count:/{print "ata_err="$4} +/Power_Cycle_Count/{print "pwr_cyc="$10} + +# SATA common +/SATA/{type="sata"} +/Temperature_Celsius/{print "temp="$10} +/Airflow_Temperature_Cel/{print "temp="$10} +/Current Temperature:/{print "temp="$3} +/SMART overall-health self-assessment test result:/{print "health="$6} +/Power_On_Hours/{print "hours_on="$10; hours_on=int($10)} +/Serial Number:/{print "serial="$3} + +# NVMe common +/NVMe/{type="nvme"} +/Temperature:/{print "temp="$2} +/SMART overall-health self-assessment test result:/{print "health="$6} +/Power On Hours:/{gsub("[^0-9]","",$4); print "hours_on="$4} +/Serial Number:/{print "serial="$3} +/Power Cycles:/{print "pwr_cyc="$3} + +# NVMe specific +/Media and Data Integrity Errors:/{print "nvme_err="$6} + +# SMART self-test info +/Self-test execution status:/{progress=tolower($4)} # SAS +/SMART Self-test log/{test_seen=1} # SAS +/SMART Extended Self-test Log/{test_seen=1} # SATA +/# 1/{ + test_type=tolower($3"_"$4); + # Status could be one word ("Completed") or multiple ("Completed: read + # failure"). Look for the ":" to see if we need to grab more words. + + if ($5 ~ ":") + status=tolower($5""$6"_"$7) + else + status=tolower($5) + if (status=="self") + status="running"; + + if (type == "sas") { + hours=int($(NF-4)) + } else { + hours=int($(NF-1)) + # SATA reports percent remaining, rather than percent done + # Convert it to percent done. + progress=(100-int($(NF-2)))"%" + } + # When we int()-ify "hours", it converts stuff like "NOW" and "-" into + # 0. In those cases, set it to hours_on, so they will cancel out in + # the "hours_ago" calculation later on. + if (hours == 0) + hours=hours_on + + if (test_seen) { + print "test="hours_on + print "test_type="test_type + print "test_status="status + print "test_progress="progress + } + # Not all drives report hours_on + if (hours_on && hours) { + total_hours_ago=(hours_on-hours) + days_ago=int(total_hours_ago/24) + hours_ago=(total_hours_ago % 24) + if (days_ago != 0) + ago_str=days_ago"d" + if (hours_ago !=0) + ago_str=ago_str""hours_ago"h" + print "test_ended="ago_str + } +} + +END {print "type="type; ORS="\n"; print ""} +'); +fi +type=$(echo "$out" | grep '^type=' | cut -d '=' -f 2) + +# If type is not set by now, either we don't have a block device +# or smartctl failed. Either way, default to ATA and set $out to +# nothing. +if [ -z "$type" ]; then + type="sata" + out= +fi + +case $script in +smart) + # Print temperature plus common predictors of drive failure + if [ "$type" = "sas" ] ; then + scripts="temp|health|r_ucor|w_ucor" + elif [ "$type" = "sata" ] ; then + scripts="temp|health|ata_err|realloc|rep_ucor|cmd_to|pend_sec|off_ucor" + elif [ "$type" = "nvme" ] ; then + scripts="temp|health|nvme_err" + fi + ;; +smartx) + # Print some other interesting stats + if [ "$type" = "sas" ] ; then + scripts="hours_on|defect|nonmed|r_proc|w_proc" + elif [ "$type" = "sata" ] ; then + scripts="hours_on|pwr_cyc" + elif [ "$type" = "nvme" ] ; then + scripts="hours_on|pwr_cyc" + fi + ;; +smart_test) + scripts="test_type|test_status|test_progress|test_ended" + ;; +*) + scripts="$script" +esac + +with_vals=$(echo "$out" | grep -E "$scripts") +if [ -n "$with_vals" ]; then + echo "$with_vals" + without_vals=$(echo "$scripts" | tr '|' '\n' | + grep -v -E "$(echo "$with_vals" | + awk -F "=" '{print $1}')" | awk '{print $0"="}') +else + without_vals=$(echo "$scripts" | tr '|' '\n' | awk '{print $0"="}') +fi + +if [ -n "$without_vals" ]; then + echo "$without_vals" +fi diff --git a/straper/db/public/zfs/etc-zfs/zpool.d/smartx b/straper/db/public/zfs/etc-zfs/zpool.d/smartx new file mode 100755 index 0000000..8ad3e10 --- /dev/null +++ b/straper/db/public/zfs/etc-zfs/zpool.d/smartx @@ -0,0 +1,249 @@ +#!/bin/sh +# +# Show SMART stats +# + +helpstr=" +smart: Show SMART temperature and error stats (specific to drive type) +smartx: Show SMART extended drive stats (specific to drive type). +temp: Show SMART drive temperature in celsius (all drives). +health: Show reported SMART status (all drives). +r_proc: Show SMART read GBytes processed over drive lifetime (SAS). +w_proc: Show SMART write GBytes processed over drive lifetime (SAS). +r_ucor: Show SMART read uncorrectable errors (SAS). +w_ucor: Show SMART write uncorrectable errors (SAS). +nonmed: Show SMART non-medium errors (SAS). +defect: Show SMART grown defect list (SAS). +hours_on: Show number of hours drive powered on (all drives). +realloc: Show SMART reallocated sectors count (ATA). +rep_ucor: Show SMART reported uncorrectable count (ATA). +cmd_to: Show SMART command timeout count (ATA). +pend_sec: Show SMART current pending sector count (ATA). +off_ucor: Show SMART offline uncorrectable errors (ATA). +ata_err: Show SMART ATA errors (ATA). +pwr_cyc: Show SMART power cycle count (ATA). +serial: Show disk serial number. +nvme_err: Show SMART NVMe errors (NVMe). +smart_test: Show SMART self-test results summary. +test_type: Show SMART self-test type (short, long... ). +test_status: Show SMART self-test status. +test_progress: Show SMART self-test percentage done. +test_ended: Show when the last SMART self-test ended (if supported). +" + +# Hack for developer testing +# +# If you set $samples to a directory containing smartctl output text files, +# we will use them instead of running smartctl on the vdevs. This can be +# useful if you want to test a bunch of different smartctl outputs. Also, if +# $samples is set, and additional 'file' column is added to the zpool output +# showing the filename. +samples= + +# get_filename_from_dir DIR +# +# Look in directory DIR and return a filename from it. The filename returned +# is chosen quasi-sequentially (based off our PID). This allows us to return +# a different filename every time this script is invoked (which we do for each +# vdev), without having to maintain state. +get_filename_from_dir() +{ + dir=$1 + pid="$$" + num_files=$(find "$dir" -maxdepth 1 -type f | wc -l) + mod=$((pid % num_files)) + i=0 + find "$dir" -type f -printf '%f\n' | while read -r file ; do + if [ "$mod" = "$i" ] ; then + echo "$file" + break + fi + i=$((i+1)) + done +} + +script="${0##*/}" + +if [ "$1" = "-h" ] ; then + echo "$helpstr" | grep "$script:" | tr -s '\t' | cut -f 2- + exit +fi + +# Sometimes, UPATH ends up /dev/(null). +# That should be corrected, but for now... +# shellcheck disable=SC2154 +if [ ! -b "$VDEV_UPATH" ]; then + somepath="${VDEV_PATH}" +else + somepath="${VDEV_UPATH}" +fi + +if [ -b "$somepath" ] && PATH="/usr/sbin:$PATH" command -v smartctl > /dev/null || [ -n "$samples" ] ; then + if [ -n "$samples" ] ; then + # cat a smartctl output text file instead of running smartctl + # on a vdev (only used for developer testing). + file=$(get_filename_from_dir "$samples") + echo "file=$file" + raw_out=$(cat "$samples/$file") + else + raw_out=$(sudo smartctl -a "$somepath") + fi + + # What kind of drive are we? Look for the right line in smartctl: + # + # SAS: + # Transport protocol: SAS + # + # SATA: + # ATA Version is: 8 + # + # NVMe: + # SMART/Health Information (NVMe Log 0xnn, NSID 0xnn) + # + out=$(echo "$raw_out" | awk ' +# SAS specific +/read:/{print "rrd="$4"\nr_cor="$5"\nr_proc="$7"\nr_ucor="$8} +/write:/{print "rwr="$4"\nw_cor="$5"\nw_proc="$7"\nw_ucor="$8} +/Non-medium error count/{print "nonmed="$4} +/Elements in grown defect list/{print "defect="$6} + +# SAS common +/SAS/{type="sas"} +/Drive Temperature:/{print "temp="$4} +# Status can be a long string, substitute spaces for '_' +/SMART Health Status:/{printf "health="; for(i=4;i<=NF-1;i++){printf "%s_", $i}; printf "%s\n", $i} +/number of hours powered up/{print "hours_on="$7; hours_on=int($7)} +/Serial number:/{print "serial="$3} + +# SATA specific +/Reallocated_Sector_Ct/{print "realloc="$10} +/Reported_Uncorrect/{print "rep_ucor="$10} +/Command_Timeout/{print "cmd_to="$10} +/Current_Pending_Sector/{print "pend_sec="$10} +/Offline_Uncorrectable/{print "off_ucor="$10} +/ATA Error Count:/{print "ata_err="$4} +/Power_Cycle_Count/{print "pwr_cyc="$10} + +# SATA common +/SATA/{type="sata"} +/Temperature_Celsius/{print "temp="$10} +/Airflow_Temperature_Cel/{print "temp="$10} +/Current Temperature:/{print "temp="$3} +/SMART overall-health self-assessment test result:/{print "health="$6} +/Power_On_Hours/{print "hours_on="$10; hours_on=int($10)} +/Serial Number:/{print "serial="$3} + +# NVMe common +/NVMe/{type="nvme"} +/Temperature:/{print "temp="$2} +/SMART overall-health self-assessment test result:/{print "health="$6} +/Power On Hours:/{gsub("[^0-9]","",$4); print "hours_on="$4} +/Serial Number:/{print "serial="$3} +/Power Cycles:/{print "pwr_cyc="$3} + +# NVMe specific +/Media and Data Integrity Errors:/{print "nvme_err="$6} + +# SMART self-test info +/Self-test execution status:/{progress=tolower($4)} # SAS +/SMART Self-test log/{test_seen=1} # SAS +/SMART Extended Self-test Log/{test_seen=1} # SATA +/# 1/{ + test_type=tolower($3"_"$4); + # Status could be one word ("Completed") or multiple ("Completed: read + # failure"). Look for the ":" to see if we need to grab more words. + + if ($5 ~ ":") + status=tolower($5""$6"_"$7) + else + status=tolower($5) + if (status=="self") + status="running"; + + if (type == "sas") { + hours=int($(NF-4)) + } else { + hours=int($(NF-1)) + # SATA reports percent remaining, rather than percent done + # Convert it to percent done. + progress=(100-int($(NF-2)))"%" + } + # When we int()-ify "hours", it converts stuff like "NOW" and "-" into + # 0. In those cases, set it to hours_on, so they will cancel out in + # the "hours_ago" calculation later on. + if (hours == 0) + hours=hours_on + + if (test_seen) { + print "test="hours_on + print "test_type="test_type + print "test_status="status + print "test_progress="progress + } + # Not all drives report hours_on + if (hours_on && hours) { + total_hours_ago=(hours_on-hours) + days_ago=int(total_hours_ago/24) + hours_ago=(total_hours_ago % 24) + if (days_ago != 0) + ago_str=days_ago"d" + if (hours_ago !=0) + ago_str=ago_str""hours_ago"h" + print "test_ended="ago_str + } +} + +END {print "type="type; ORS="\n"; print ""} +'); +fi +type=$(echo "$out" | grep '^type=' | cut -d '=' -f 2) + +# If type is not set by now, either we don't have a block device +# or smartctl failed. Either way, default to ATA and set $out to +# nothing. +if [ -z "$type" ]; then + type="sata" + out= +fi + +case $script in +smart) + # Print temperature plus common predictors of drive failure + if [ "$type" = "sas" ] ; then + scripts="temp|health|r_ucor|w_ucor" + elif [ "$type" = "sata" ] ; then + scripts="temp|health|ata_err|realloc|rep_ucor|cmd_to|pend_sec|off_ucor" + elif [ "$type" = "nvme" ] ; then + scripts="temp|health|nvme_err" + fi + ;; +smartx) + # Print some other interesting stats + if [ "$type" = "sas" ] ; then + scripts="hours_on|defect|nonmed|r_proc|w_proc" + elif [ "$type" = "sata" ] ; then + scripts="hours_on|pwr_cyc" + elif [ "$type" = "nvme" ] ; then + scripts="hours_on|pwr_cyc" + fi + ;; +smart_test) + scripts="test_type|test_status|test_progress|test_ended" + ;; +*) + scripts="$script" +esac + +with_vals=$(echo "$out" | grep -E "$scripts") +if [ -n "$with_vals" ]; then + echo "$with_vals" + without_vals=$(echo "$scripts" | tr '|' '\n' | + grep -v -E "$(echo "$with_vals" | + awk -F "=" '{print $1}')" | awk '{print $0"="}') +else + without_vals=$(echo "$scripts" | tr '|' '\n' | awk '{print $0"="}') +fi + +if [ -n "$without_vals" ]; then + echo "$without_vals" +fi diff --git a/straper/db/public/zfs/etc-zfs/zpool.d/temp b/straper/db/public/zfs/etc-zfs/zpool.d/temp new file mode 100755 index 0000000..8ad3e10 --- /dev/null +++ b/straper/db/public/zfs/etc-zfs/zpool.d/temp @@ -0,0 +1,249 @@ +#!/bin/sh +# +# Show SMART stats +# + +helpstr=" +smart: Show SMART temperature and error stats (specific to drive type) +smartx: Show SMART extended drive stats (specific to drive type). +temp: Show SMART drive temperature in celsius (all drives). +health: Show reported SMART status (all drives). +r_proc: Show SMART read GBytes processed over drive lifetime (SAS). +w_proc: Show SMART write GBytes processed over drive lifetime (SAS). +r_ucor: Show SMART read uncorrectable errors (SAS). +w_ucor: Show SMART write uncorrectable errors (SAS). +nonmed: Show SMART non-medium errors (SAS). +defect: Show SMART grown defect list (SAS). +hours_on: Show number of hours drive powered on (all drives). +realloc: Show SMART reallocated sectors count (ATA). +rep_ucor: Show SMART reported uncorrectable count (ATA). +cmd_to: Show SMART command timeout count (ATA). +pend_sec: Show SMART current pending sector count (ATA). +off_ucor: Show SMART offline uncorrectable errors (ATA). +ata_err: Show SMART ATA errors (ATA). +pwr_cyc: Show SMART power cycle count (ATA). +serial: Show disk serial number. +nvme_err: Show SMART NVMe errors (NVMe). +smart_test: Show SMART self-test results summary. +test_type: Show SMART self-test type (short, long... ). +test_status: Show SMART self-test status. +test_progress: Show SMART self-test percentage done. +test_ended: Show when the last SMART self-test ended (if supported). +" + +# Hack for developer testing +# +# If you set $samples to a directory containing smartctl output text files, +# we will use them instead of running smartctl on the vdevs. This can be +# useful if you want to test a bunch of different smartctl outputs. Also, if +# $samples is set, and additional 'file' column is added to the zpool output +# showing the filename. +samples= + +# get_filename_from_dir DIR +# +# Look in directory DIR and return a filename from it. The filename returned +# is chosen quasi-sequentially (based off our PID). This allows us to return +# a different filename every time this script is invoked (which we do for each +# vdev), without having to maintain state. +get_filename_from_dir() +{ + dir=$1 + pid="$$" + num_files=$(find "$dir" -maxdepth 1 -type f | wc -l) + mod=$((pid % num_files)) + i=0 + find "$dir" -type f -printf '%f\n' | while read -r file ; do + if [ "$mod" = "$i" ] ; then + echo "$file" + break + fi + i=$((i+1)) + done +} + +script="${0##*/}" + +if [ "$1" = "-h" ] ; then + echo "$helpstr" | grep "$script:" | tr -s '\t' | cut -f 2- + exit +fi + +# Sometimes, UPATH ends up /dev/(null). +# That should be corrected, but for now... +# shellcheck disable=SC2154 +if [ ! -b "$VDEV_UPATH" ]; then + somepath="${VDEV_PATH}" +else + somepath="${VDEV_UPATH}" +fi + +if [ -b "$somepath" ] && PATH="/usr/sbin:$PATH" command -v smartctl > /dev/null || [ -n "$samples" ] ; then + if [ -n "$samples" ] ; then + # cat a smartctl output text file instead of running smartctl + # on a vdev (only used for developer testing). + file=$(get_filename_from_dir "$samples") + echo "file=$file" + raw_out=$(cat "$samples/$file") + else + raw_out=$(sudo smartctl -a "$somepath") + fi + + # What kind of drive are we? Look for the right line in smartctl: + # + # SAS: + # Transport protocol: SAS + # + # SATA: + # ATA Version is: 8 + # + # NVMe: + # SMART/Health Information (NVMe Log 0xnn, NSID 0xnn) + # + out=$(echo "$raw_out" | awk ' +# SAS specific +/read:/{print "rrd="$4"\nr_cor="$5"\nr_proc="$7"\nr_ucor="$8} +/write:/{print "rwr="$4"\nw_cor="$5"\nw_proc="$7"\nw_ucor="$8} +/Non-medium error count/{print "nonmed="$4} +/Elements in grown defect list/{print "defect="$6} + +# SAS common +/SAS/{type="sas"} +/Drive Temperature:/{print "temp="$4} +# Status can be a long string, substitute spaces for '_' +/SMART Health Status:/{printf "health="; for(i=4;i<=NF-1;i++){printf "%s_", $i}; printf "%s\n", $i} +/number of hours powered up/{print "hours_on="$7; hours_on=int($7)} +/Serial number:/{print "serial="$3} + +# SATA specific +/Reallocated_Sector_Ct/{print "realloc="$10} +/Reported_Uncorrect/{print "rep_ucor="$10} +/Command_Timeout/{print "cmd_to="$10} +/Current_Pending_Sector/{print "pend_sec="$10} +/Offline_Uncorrectable/{print "off_ucor="$10} +/ATA Error Count:/{print "ata_err="$4} +/Power_Cycle_Count/{print "pwr_cyc="$10} + +# SATA common +/SATA/{type="sata"} +/Temperature_Celsius/{print "temp="$10} +/Airflow_Temperature_Cel/{print "temp="$10} +/Current Temperature:/{print "temp="$3} +/SMART overall-health self-assessment test result:/{print "health="$6} +/Power_On_Hours/{print "hours_on="$10; hours_on=int($10)} +/Serial Number:/{print "serial="$3} + +# NVMe common +/NVMe/{type="nvme"} +/Temperature:/{print "temp="$2} +/SMART overall-health self-assessment test result:/{print "health="$6} +/Power On Hours:/{gsub("[^0-9]","",$4); print "hours_on="$4} +/Serial Number:/{print "serial="$3} +/Power Cycles:/{print "pwr_cyc="$3} + +# NVMe specific +/Media and Data Integrity Errors:/{print "nvme_err="$6} + +# SMART self-test info +/Self-test execution status:/{progress=tolower($4)} # SAS +/SMART Self-test log/{test_seen=1} # SAS +/SMART Extended Self-test Log/{test_seen=1} # SATA +/# 1/{ + test_type=tolower($3"_"$4); + # Status could be one word ("Completed") or multiple ("Completed: read + # failure"). Look for the ":" to see if we need to grab more words. + + if ($5 ~ ":") + status=tolower($5""$6"_"$7) + else + status=tolower($5) + if (status=="self") + status="running"; + + if (type == "sas") { + hours=int($(NF-4)) + } else { + hours=int($(NF-1)) + # SATA reports percent remaining, rather than percent done + # Convert it to percent done. + progress=(100-int($(NF-2)))"%" + } + # When we int()-ify "hours", it converts stuff like "NOW" and "-" into + # 0. In those cases, set it to hours_on, so they will cancel out in + # the "hours_ago" calculation later on. + if (hours == 0) + hours=hours_on + + if (test_seen) { + print "test="hours_on + print "test_type="test_type + print "test_status="status + print "test_progress="progress + } + # Not all drives report hours_on + if (hours_on && hours) { + total_hours_ago=(hours_on-hours) + days_ago=int(total_hours_ago/24) + hours_ago=(total_hours_ago % 24) + if (days_ago != 0) + ago_str=days_ago"d" + if (hours_ago !=0) + ago_str=ago_str""hours_ago"h" + print "test_ended="ago_str + } +} + +END {print "type="type; ORS="\n"; print ""} +'); +fi +type=$(echo "$out" | grep '^type=' | cut -d '=' -f 2) + +# If type is not set by now, either we don't have a block device +# or smartctl failed. Either way, default to ATA and set $out to +# nothing. +if [ -z "$type" ]; then + type="sata" + out= +fi + +case $script in +smart) + # Print temperature plus common predictors of drive failure + if [ "$type" = "sas" ] ; then + scripts="temp|health|r_ucor|w_ucor" + elif [ "$type" = "sata" ] ; then + scripts="temp|health|ata_err|realloc|rep_ucor|cmd_to|pend_sec|off_ucor" + elif [ "$type" = "nvme" ] ; then + scripts="temp|health|nvme_err" + fi + ;; +smartx) + # Print some other interesting stats + if [ "$type" = "sas" ] ; then + scripts="hours_on|defect|nonmed|r_proc|w_proc" + elif [ "$type" = "sata" ] ; then + scripts="hours_on|pwr_cyc" + elif [ "$type" = "nvme" ] ; then + scripts="hours_on|pwr_cyc" + fi + ;; +smart_test) + scripts="test_type|test_status|test_progress|test_ended" + ;; +*) + scripts="$script" +esac + +with_vals=$(echo "$out" | grep -E "$scripts") +if [ -n "$with_vals" ]; then + echo "$with_vals" + without_vals=$(echo "$scripts" | tr '|' '\n' | + grep -v -E "$(echo "$with_vals" | + awk -F "=" '{print $1}')" | awk '{print $0"="}') +else + without_vals=$(echo "$scripts" | tr '|' '\n' | awk '{print $0"="}') +fi + +if [ -n "$without_vals" ]; then + echo "$without_vals" +fi diff --git a/straper/db/public/zfs/etc-zfs/zpool.d/test_ended b/straper/db/public/zfs/etc-zfs/zpool.d/test_ended new file mode 100755 index 0000000..8ad3e10 --- /dev/null +++ b/straper/db/public/zfs/etc-zfs/zpool.d/test_ended @@ -0,0 +1,249 @@ +#!/bin/sh +# +# Show SMART stats +# + +helpstr=" +smart: Show SMART temperature and error stats (specific to drive type) +smartx: Show SMART extended drive stats (specific to drive type). +temp: Show SMART drive temperature in celsius (all drives). +health: Show reported SMART status (all drives). +r_proc: Show SMART read GBytes processed over drive lifetime (SAS). +w_proc: Show SMART write GBytes processed over drive lifetime (SAS). +r_ucor: Show SMART read uncorrectable errors (SAS). +w_ucor: Show SMART write uncorrectable errors (SAS). +nonmed: Show SMART non-medium errors (SAS). +defect: Show SMART grown defect list (SAS). +hours_on: Show number of hours drive powered on (all drives). +realloc: Show SMART reallocated sectors count (ATA). +rep_ucor: Show SMART reported uncorrectable count (ATA). +cmd_to: Show SMART command timeout count (ATA). +pend_sec: Show SMART current pending sector count (ATA). +off_ucor: Show SMART offline uncorrectable errors (ATA). +ata_err: Show SMART ATA errors (ATA). +pwr_cyc: Show SMART power cycle count (ATA). +serial: Show disk serial number. +nvme_err: Show SMART NVMe errors (NVMe). +smart_test: Show SMART self-test results summary. +test_type: Show SMART self-test type (short, long... ). +test_status: Show SMART self-test status. +test_progress: Show SMART self-test percentage done. +test_ended: Show when the last SMART self-test ended (if supported). +" + +# Hack for developer testing +# +# If you set $samples to a directory containing smartctl output text files, +# we will use them instead of running smartctl on the vdevs. This can be +# useful if you want to test a bunch of different smartctl outputs. Also, if +# $samples is set, and additional 'file' column is added to the zpool output +# showing the filename. +samples= + +# get_filename_from_dir DIR +# +# Look in directory DIR and return a filename from it. The filename returned +# is chosen quasi-sequentially (based off our PID). This allows us to return +# a different filename every time this script is invoked (which we do for each +# vdev), without having to maintain state. +get_filename_from_dir() +{ + dir=$1 + pid="$$" + num_files=$(find "$dir" -maxdepth 1 -type f | wc -l) + mod=$((pid % num_files)) + i=0 + find "$dir" -type f -printf '%f\n' | while read -r file ; do + if [ "$mod" = "$i" ] ; then + echo "$file" + break + fi + i=$((i+1)) + done +} + +script="${0##*/}" + +if [ "$1" = "-h" ] ; then + echo "$helpstr" | grep "$script:" | tr -s '\t' | cut -f 2- + exit +fi + +# Sometimes, UPATH ends up /dev/(null). +# That should be corrected, but for now... +# shellcheck disable=SC2154 +if [ ! -b "$VDEV_UPATH" ]; then + somepath="${VDEV_PATH}" +else + somepath="${VDEV_UPATH}" +fi + +if [ -b "$somepath" ] && PATH="/usr/sbin:$PATH" command -v smartctl > /dev/null || [ -n "$samples" ] ; then + if [ -n "$samples" ] ; then + # cat a smartctl output text file instead of running smartctl + # on a vdev (only used for developer testing). + file=$(get_filename_from_dir "$samples") + echo "file=$file" + raw_out=$(cat "$samples/$file") + else + raw_out=$(sudo smartctl -a "$somepath") + fi + + # What kind of drive are we? Look for the right line in smartctl: + # + # SAS: + # Transport protocol: SAS + # + # SATA: + # ATA Version is: 8 + # + # NVMe: + # SMART/Health Information (NVMe Log 0xnn, NSID 0xnn) + # + out=$(echo "$raw_out" | awk ' +# SAS specific +/read:/{print "rrd="$4"\nr_cor="$5"\nr_proc="$7"\nr_ucor="$8} +/write:/{print "rwr="$4"\nw_cor="$5"\nw_proc="$7"\nw_ucor="$8} +/Non-medium error count/{print "nonmed="$4} +/Elements in grown defect list/{print "defect="$6} + +# SAS common +/SAS/{type="sas"} +/Drive Temperature:/{print "temp="$4} +# Status can be a long string, substitute spaces for '_' +/SMART Health Status:/{printf "health="; for(i=4;i<=NF-1;i++){printf "%s_", $i}; printf "%s\n", $i} +/number of hours powered up/{print "hours_on="$7; hours_on=int($7)} +/Serial number:/{print "serial="$3} + +# SATA specific +/Reallocated_Sector_Ct/{print "realloc="$10} +/Reported_Uncorrect/{print "rep_ucor="$10} +/Command_Timeout/{print "cmd_to="$10} +/Current_Pending_Sector/{print "pend_sec="$10} +/Offline_Uncorrectable/{print "off_ucor="$10} +/ATA Error Count:/{print "ata_err="$4} +/Power_Cycle_Count/{print "pwr_cyc="$10} + +# SATA common +/SATA/{type="sata"} +/Temperature_Celsius/{print "temp="$10} +/Airflow_Temperature_Cel/{print "temp="$10} +/Current Temperature:/{print "temp="$3} +/SMART overall-health self-assessment test result:/{print "health="$6} +/Power_On_Hours/{print "hours_on="$10; hours_on=int($10)} +/Serial Number:/{print "serial="$3} + +# NVMe common +/NVMe/{type="nvme"} +/Temperature:/{print "temp="$2} +/SMART overall-health self-assessment test result:/{print "health="$6} +/Power On Hours:/{gsub("[^0-9]","",$4); print "hours_on="$4} +/Serial Number:/{print "serial="$3} +/Power Cycles:/{print "pwr_cyc="$3} + +# NVMe specific +/Media and Data Integrity Errors:/{print "nvme_err="$6} + +# SMART self-test info +/Self-test execution status:/{progress=tolower($4)} # SAS +/SMART Self-test log/{test_seen=1} # SAS +/SMART Extended Self-test Log/{test_seen=1} # SATA +/# 1/{ + test_type=tolower($3"_"$4); + # Status could be one word ("Completed") or multiple ("Completed: read + # failure"). Look for the ":" to see if we need to grab more words. + + if ($5 ~ ":") + status=tolower($5""$6"_"$7) + else + status=tolower($5) + if (status=="self") + status="running"; + + if (type == "sas") { + hours=int($(NF-4)) + } else { + hours=int($(NF-1)) + # SATA reports percent remaining, rather than percent done + # Convert it to percent done. + progress=(100-int($(NF-2)))"%" + } + # When we int()-ify "hours", it converts stuff like "NOW" and "-" into + # 0. In those cases, set it to hours_on, so they will cancel out in + # the "hours_ago" calculation later on. + if (hours == 0) + hours=hours_on + + if (test_seen) { + print "test="hours_on + print "test_type="test_type + print "test_status="status + print "test_progress="progress + } + # Not all drives report hours_on + if (hours_on && hours) { + total_hours_ago=(hours_on-hours) + days_ago=int(total_hours_ago/24) + hours_ago=(total_hours_ago % 24) + if (days_ago != 0) + ago_str=days_ago"d" + if (hours_ago !=0) + ago_str=ago_str""hours_ago"h" + print "test_ended="ago_str + } +} + +END {print "type="type; ORS="\n"; print ""} +'); +fi +type=$(echo "$out" | grep '^type=' | cut -d '=' -f 2) + +# If type is not set by now, either we don't have a block device +# or smartctl failed. Either way, default to ATA and set $out to +# nothing. +if [ -z "$type" ]; then + type="sata" + out= +fi + +case $script in +smart) + # Print temperature plus common predictors of drive failure + if [ "$type" = "sas" ] ; then + scripts="temp|health|r_ucor|w_ucor" + elif [ "$type" = "sata" ] ; then + scripts="temp|health|ata_err|realloc|rep_ucor|cmd_to|pend_sec|off_ucor" + elif [ "$type" = "nvme" ] ; then + scripts="temp|health|nvme_err" + fi + ;; +smartx) + # Print some other interesting stats + if [ "$type" = "sas" ] ; then + scripts="hours_on|defect|nonmed|r_proc|w_proc" + elif [ "$type" = "sata" ] ; then + scripts="hours_on|pwr_cyc" + elif [ "$type" = "nvme" ] ; then + scripts="hours_on|pwr_cyc" + fi + ;; +smart_test) + scripts="test_type|test_status|test_progress|test_ended" + ;; +*) + scripts="$script" +esac + +with_vals=$(echo "$out" | grep -E "$scripts") +if [ -n "$with_vals" ]; then + echo "$with_vals" + without_vals=$(echo "$scripts" | tr '|' '\n' | + grep -v -E "$(echo "$with_vals" | + awk -F "=" '{print $1}')" | awk '{print $0"="}') +else + without_vals=$(echo "$scripts" | tr '|' '\n' | awk '{print $0"="}') +fi + +if [ -n "$without_vals" ]; then + echo "$without_vals" +fi diff --git a/straper/db/public/zfs/etc-zfs/zpool.d/test_progress b/straper/db/public/zfs/etc-zfs/zpool.d/test_progress new file mode 100755 index 0000000..8ad3e10 --- /dev/null +++ b/straper/db/public/zfs/etc-zfs/zpool.d/test_progress @@ -0,0 +1,249 @@ +#!/bin/sh +# +# Show SMART stats +# + +helpstr=" +smart: Show SMART temperature and error stats (specific to drive type) +smartx: Show SMART extended drive stats (specific to drive type). +temp: Show SMART drive temperature in celsius (all drives). +health: Show reported SMART status (all drives). +r_proc: Show SMART read GBytes processed over drive lifetime (SAS). +w_proc: Show SMART write GBytes processed over drive lifetime (SAS). +r_ucor: Show SMART read uncorrectable errors (SAS). +w_ucor: Show SMART write uncorrectable errors (SAS). +nonmed: Show SMART non-medium errors (SAS). +defect: Show SMART grown defect list (SAS). +hours_on: Show number of hours drive powered on (all drives). +realloc: Show SMART reallocated sectors count (ATA). +rep_ucor: Show SMART reported uncorrectable count (ATA). +cmd_to: Show SMART command timeout count (ATA). +pend_sec: Show SMART current pending sector count (ATA). +off_ucor: Show SMART offline uncorrectable errors (ATA). +ata_err: Show SMART ATA errors (ATA). +pwr_cyc: Show SMART power cycle count (ATA). +serial: Show disk serial number. +nvme_err: Show SMART NVMe errors (NVMe). +smart_test: Show SMART self-test results summary. +test_type: Show SMART self-test type (short, long... ). +test_status: Show SMART self-test status. +test_progress: Show SMART self-test percentage done. +test_ended: Show when the last SMART self-test ended (if supported). +" + +# Hack for developer testing +# +# If you set $samples to a directory containing smartctl output text files, +# we will use them instead of running smartctl on the vdevs. This can be +# useful if you want to test a bunch of different smartctl outputs. Also, if +# $samples is set, and additional 'file' column is added to the zpool output +# showing the filename. +samples= + +# get_filename_from_dir DIR +# +# Look in directory DIR and return a filename from it. The filename returned +# is chosen quasi-sequentially (based off our PID). This allows us to return +# a different filename every time this script is invoked (which we do for each +# vdev), without having to maintain state. +get_filename_from_dir() +{ + dir=$1 + pid="$$" + num_files=$(find "$dir" -maxdepth 1 -type f | wc -l) + mod=$((pid % num_files)) + i=0 + find "$dir" -type f -printf '%f\n' | while read -r file ; do + if [ "$mod" = "$i" ] ; then + echo "$file" + break + fi + i=$((i+1)) + done +} + +script="${0##*/}" + +if [ "$1" = "-h" ] ; then + echo "$helpstr" | grep "$script:" | tr -s '\t' | cut -f 2- + exit +fi + +# Sometimes, UPATH ends up /dev/(null). +# That should be corrected, but for now... +# shellcheck disable=SC2154 +if [ ! -b "$VDEV_UPATH" ]; then + somepath="${VDEV_PATH}" +else + somepath="${VDEV_UPATH}" +fi + +if [ -b "$somepath" ] && PATH="/usr/sbin:$PATH" command -v smartctl > /dev/null || [ -n "$samples" ] ; then + if [ -n "$samples" ] ; then + # cat a smartctl output text file instead of running smartctl + # on a vdev (only used for developer testing). + file=$(get_filename_from_dir "$samples") + echo "file=$file" + raw_out=$(cat "$samples/$file") + else + raw_out=$(sudo smartctl -a "$somepath") + fi + + # What kind of drive are we? Look for the right line in smartctl: + # + # SAS: + # Transport protocol: SAS + # + # SATA: + # ATA Version is: 8 + # + # NVMe: + # SMART/Health Information (NVMe Log 0xnn, NSID 0xnn) + # + out=$(echo "$raw_out" | awk ' +# SAS specific +/read:/{print "rrd="$4"\nr_cor="$5"\nr_proc="$7"\nr_ucor="$8} +/write:/{print "rwr="$4"\nw_cor="$5"\nw_proc="$7"\nw_ucor="$8} +/Non-medium error count/{print "nonmed="$4} +/Elements in grown defect list/{print "defect="$6} + +# SAS common +/SAS/{type="sas"} +/Drive Temperature:/{print "temp="$4} +# Status can be a long string, substitute spaces for '_' +/SMART Health Status:/{printf "health="; for(i=4;i<=NF-1;i++){printf "%s_", $i}; printf "%s\n", $i} +/number of hours powered up/{print "hours_on="$7; hours_on=int($7)} +/Serial number:/{print "serial="$3} + +# SATA specific +/Reallocated_Sector_Ct/{print "realloc="$10} +/Reported_Uncorrect/{print "rep_ucor="$10} +/Command_Timeout/{print "cmd_to="$10} +/Current_Pending_Sector/{print "pend_sec="$10} +/Offline_Uncorrectable/{print "off_ucor="$10} +/ATA Error Count:/{print "ata_err="$4} +/Power_Cycle_Count/{print "pwr_cyc="$10} + +# SATA common +/SATA/{type="sata"} +/Temperature_Celsius/{print "temp="$10} +/Airflow_Temperature_Cel/{print "temp="$10} +/Current Temperature:/{print "temp="$3} +/SMART overall-health self-assessment test result:/{print "health="$6} +/Power_On_Hours/{print "hours_on="$10; hours_on=int($10)} +/Serial Number:/{print "serial="$3} + +# NVMe common +/NVMe/{type="nvme"} +/Temperature:/{print "temp="$2} +/SMART overall-health self-assessment test result:/{print "health="$6} +/Power On Hours:/{gsub("[^0-9]","",$4); print "hours_on="$4} +/Serial Number:/{print "serial="$3} +/Power Cycles:/{print "pwr_cyc="$3} + +# NVMe specific +/Media and Data Integrity Errors:/{print "nvme_err="$6} + +# SMART self-test info +/Self-test execution status:/{progress=tolower($4)} # SAS +/SMART Self-test log/{test_seen=1} # SAS +/SMART Extended Self-test Log/{test_seen=1} # SATA +/# 1/{ + test_type=tolower($3"_"$4); + # Status could be one word ("Completed") or multiple ("Completed: read + # failure"). Look for the ":" to see if we need to grab more words. + + if ($5 ~ ":") + status=tolower($5""$6"_"$7) + else + status=tolower($5) + if (status=="self") + status="running"; + + if (type == "sas") { + hours=int($(NF-4)) + } else { + hours=int($(NF-1)) + # SATA reports percent remaining, rather than percent done + # Convert it to percent done. + progress=(100-int($(NF-2)))"%" + } + # When we int()-ify "hours", it converts stuff like "NOW" and "-" into + # 0. In those cases, set it to hours_on, so they will cancel out in + # the "hours_ago" calculation later on. + if (hours == 0) + hours=hours_on + + if (test_seen) { + print "test="hours_on + print "test_type="test_type + print "test_status="status + print "test_progress="progress + } + # Not all drives report hours_on + if (hours_on && hours) { + total_hours_ago=(hours_on-hours) + days_ago=int(total_hours_ago/24) + hours_ago=(total_hours_ago % 24) + if (days_ago != 0) + ago_str=days_ago"d" + if (hours_ago !=0) + ago_str=ago_str""hours_ago"h" + print "test_ended="ago_str + } +} + +END {print "type="type; ORS="\n"; print ""} +'); +fi +type=$(echo "$out" | grep '^type=' | cut -d '=' -f 2) + +# If type is not set by now, either we don't have a block device +# or smartctl failed. Either way, default to ATA and set $out to +# nothing. +if [ -z "$type" ]; then + type="sata" + out= +fi + +case $script in +smart) + # Print temperature plus common predictors of drive failure + if [ "$type" = "sas" ] ; then + scripts="temp|health|r_ucor|w_ucor" + elif [ "$type" = "sata" ] ; then + scripts="temp|health|ata_err|realloc|rep_ucor|cmd_to|pend_sec|off_ucor" + elif [ "$type" = "nvme" ] ; then + scripts="temp|health|nvme_err" + fi + ;; +smartx) + # Print some other interesting stats + if [ "$type" = "sas" ] ; then + scripts="hours_on|defect|nonmed|r_proc|w_proc" + elif [ "$type" = "sata" ] ; then + scripts="hours_on|pwr_cyc" + elif [ "$type" = "nvme" ] ; then + scripts="hours_on|pwr_cyc" + fi + ;; +smart_test) + scripts="test_type|test_status|test_progress|test_ended" + ;; +*) + scripts="$script" +esac + +with_vals=$(echo "$out" | grep -E "$scripts") +if [ -n "$with_vals" ]; then + echo "$with_vals" + without_vals=$(echo "$scripts" | tr '|' '\n' | + grep -v -E "$(echo "$with_vals" | + awk -F "=" '{print $1}')" | awk '{print $0"="}') +else + without_vals=$(echo "$scripts" | tr '|' '\n' | awk '{print $0"="}') +fi + +if [ -n "$without_vals" ]; then + echo "$without_vals" +fi diff --git a/straper/db/public/zfs/etc-zfs/zpool.d/test_status b/straper/db/public/zfs/etc-zfs/zpool.d/test_status new file mode 100755 index 0000000..8ad3e10 --- /dev/null +++ b/straper/db/public/zfs/etc-zfs/zpool.d/test_status @@ -0,0 +1,249 @@ +#!/bin/sh +# +# Show SMART stats +# + +helpstr=" +smart: Show SMART temperature and error stats (specific to drive type) +smartx: Show SMART extended drive stats (specific to drive type). +temp: Show SMART drive temperature in celsius (all drives). +health: Show reported SMART status (all drives). +r_proc: Show SMART read GBytes processed over drive lifetime (SAS). +w_proc: Show SMART write GBytes processed over drive lifetime (SAS). +r_ucor: Show SMART read uncorrectable errors (SAS). +w_ucor: Show SMART write uncorrectable errors (SAS). +nonmed: Show SMART non-medium errors (SAS). +defect: Show SMART grown defect list (SAS). +hours_on: Show number of hours drive powered on (all drives). +realloc: Show SMART reallocated sectors count (ATA). +rep_ucor: Show SMART reported uncorrectable count (ATA). +cmd_to: Show SMART command timeout count (ATA). +pend_sec: Show SMART current pending sector count (ATA). +off_ucor: Show SMART offline uncorrectable errors (ATA). +ata_err: Show SMART ATA errors (ATA). +pwr_cyc: Show SMART power cycle count (ATA). +serial: Show disk serial number. +nvme_err: Show SMART NVMe errors (NVMe). +smart_test: Show SMART self-test results summary. +test_type: Show SMART self-test type (short, long... ). +test_status: Show SMART self-test status. +test_progress: Show SMART self-test percentage done. +test_ended: Show when the last SMART self-test ended (if supported). +" + +# Hack for developer testing +# +# If you set $samples to a directory containing smartctl output text files, +# we will use them instead of running smartctl on the vdevs. This can be +# useful if you want to test a bunch of different smartctl outputs. Also, if +# $samples is set, and additional 'file' column is added to the zpool output +# showing the filename. +samples= + +# get_filename_from_dir DIR +# +# Look in directory DIR and return a filename from it. The filename returned +# is chosen quasi-sequentially (based off our PID). This allows us to return +# a different filename every time this script is invoked (which we do for each +# vdev), without having to maintain state. +get_filename_from_dir() +{ + dir=$1 + pid="$$" + num_files=$(find "$dir" -maxdepth 1 -type f | wc -l) + mod=$((pid % num_files)) + i=0 + find "$dir" -type f -printf '%f\n' | while read -r file ; do + if [ "$mod" = "$i" ] ; then + echo "$file" + break + fi + i=$((i+1)) + done +} + +script="${0##*/}" + +if [ "$1" = "-h" ] ; then + echo "$helpstr" | grep "$script:" | tr -s '\t' | cut -f 2- + exit +fi + +# Sometimes, UPATH ends up /dev/(null). +# That should be corrected, but for now... +# shellcheck disable=SC2154 +if [ ! -b "$VDEV_UPATH" ]; then + somepath="${VDEV_PATH}" +else + somepath="${VDEV_UPATH}" +fi + +if [ -b "$somepath" ] && PATH="/usr/sbin:$PATH" command -v smartctl > /dev/null || [ -n "$samples" ] ; then + if [ -n "$samples" ] ; then + # cat a smartctl output text file instead of running smartctl + # on a vdev (only used for developer testing). + file=$(get_filename_from_dir "$samples") + echo "file=$file" + raw_out=$(cat "$samples/$file") + else + raw_out=$(sudo smartctl -a "$somepath") + fi + + # What kind of drive are we? Look for the right line in smartctl: + # + # SAS: + # Transport protocol: SAS + # + # SATA: + # ATA Version is: 8 + # + # NVMe: + # SMART/Health Information (NVMe Log 0xnn, NSID 0xnn) + # + out=$(echo "$raw_out" | awk ' +# SAS specific +/read:/{print "rrd="$4"\nr_cor="$5"\nr_proc="$7"\nr_ucor="$8} +/write:/{print "rwr="$4"\nw_cor="$5"\nw_proc="$7"\nw_ucor="$8} +/Non-medium error count/{print "nonmed="$4} +/Elements in grown defect list/{print "defect="$6} + +# SAS common +/SAS/{type="sas"} +/Drive Temperature:/{print "temp="$4} +# Status can be a long string, substitute spaces for '_' +/SMART Health Status:/{printf "health="; for(i=4;i<=NF-1;i++){printf "%s_", $i}; printf "%s\n", $i} +/number of hours powered up/{print "hours_on="$7; hours_on=int($7)} +/Serial number:/{print "serial="$3} + +# SATA specific +/Reallocated_Sector_Ct/{print "realloc="$10} +/Reported_Uncorrect/{print "rep_ucor="$10} +/Command_Timeout/{print "cmd_to="$10} +/Current_Pending_Sector/{print "pend_sec="$10} +/Offline_Uncorrectable/{print "off_ucor="$10} +/ATA Error Count:/{print "ata_err="$4} +/Power_Cycle_Count/{print "pwr_cyc="$10} + +# SATA common +/SATA/{type="sata"} +/Temperature_Celsius/{print "temp="$10} +/Airflow_Temperature_Cel/{print "temp="$10} +/Current Temperature:/{print "temp="$3} +/SMART overall-health self-assessment test result:/{print "health="$6} +/Power_On_Hours/{print "hours_on="$10; hours_on=int($10)} +/Serial Number:/{print "serial="$3} + +# NVMe common +/NVMe/{type="nvme"} +/Temperature:/{print "temp="$2} +/SMART overall-health self-assessment test result:/{print "health="$6} +/Power On Hours:/{gsub("[^0-9]","",$4); print "hours_on="$4} +/Serial Number:/{print "serial="$3} +/Power Cycles:/{print "pwr_cyc="$3} + +# NVMe specific +/Media and Data Integrity Errors:/{print "nvme_err="$6} + +# SMART self-test info +/Self-test execution status:/{progress=tolower($4)} # SAS +/SMART Self-test log/{test_seen=1} # SAS +/SMART Extended Self-test Log/{test_seen=1} # SATA +/# 1/{ + test_type=tolower($3"_"$4); + # Status could be one word ("Completed") or multiple ("Completed: read + # failure"). Look for the ":" to see if we need to grab more words. + + if ($5 ~ ":") + status=tolower($5""$6"_"$7) + else + status=tolower($5) + if (status=="self") + status="running"; + + if (type == "sas") { + hours=int($(NF-4)) + } else { + hours=int($(NF-1)) + # SATA reports percent remaining, rather than percent done + # Convert it to percent done. + progress=(100-int($(NF-2)))"%" + } + # When we int()-ify "hours", it converts stuff like "NOW" and "-" into + # 0. In those cases, set it to hours_on, so they will cancel out in + # the "hours_ago" calculation later on. + if (hours == 0) + hours=hours_on + + if (test_seen) { + print "test="hours_on + print "test_type="test_type + print "test_status="status + print "test_progress="progress + } + # Not all drives report hours_on + if (hours_on && hours) { + total_hours_ago=(hours_on-hours) + days_ago=int(total_hours_ago/24) + hours_ago=(total_hours_ago % 24) + if (days_ago != 0) + ago_str=days_ago"d" + if (hours_ago !=0) + ago_str=ago_str""hours_ago"h" + print "test_ended="ago_str + } +} + +END {print "type="type; ORS="\n"; print ""} +'); +fi +type=$(echo "$out" | grep '^type=' | cut -d '=' -f 2) + +# If type is not set by now, either we don't have a block device +# or smartctl failed. Either way, default to ATA and set $out to +# nothing. +if [ -z "$type" ]; then + type="sata" + out= +fi + +case $script in +smart) + # Print temperature plus common predictors of drive failure + if [ "$type" = "sas" ] ; then + scripts="temp|health|r_ucor|w_ucor" + elif [ "$type" = "sata" ] ; then + scripts="temp|health|ata_err|realloc|rep_ucor|cmd_to|pend_sec|off_ucor" + elif [ "$type" = "nvme" ] ; then + scripts="temp|health|nvme_err" + fi + ;; +smartx) + # Print some other interesting stats + if [ "$type" = "sas" ] ; then + scripts="hours_on|defect|nonmed|r_proc|w_proc" + elif [ "$type" = "sata" ] ; then + scripts="hours_on|pwr_cyc" + elif [ "$type" = "nvme" ] ; then + scripts="hours_on|pwr_cyc" + fi + ;; +smart_test) + scripts="test_type|test_status|test_progress|test_ended" + ;; +*) + scripts="$script" +esac + +with_vals=$(echo "$out" | grep -E "$scripts") +if [ -n "$with_vals" ]; then + echo "$with_vals" + without_vals=$(echo "$scripts" | tr '|' '\n' | + grep -v -E "$(echo "$with_vals" | + awk -F "=" '{print $1}')" | awk '{print $0"="}') +else + without_vals=$(echo "$scripts" | tr '|' '\n' | awk '{print $0"="}') +fi + +if [ -n "$without_vals" ]; then + echo "$without_vals" +fi diff --git a/straper/db/public/zfs/etc-zfs/zpool.d/test_type b/straper/db/public/zfs/etc-zfs/zpool.d/test_type new file mode 100755 index 0000000..8ad3e10 --- /dev/null +++ b/straper/db/public/zfs/etc-zfs/zpool.d/test_type @@ -0,0 +1,249 @@ +#!/bin/sh +# +# Show SMART stats +# + +helpstr=" +smart: Show SMART temperature and error stats (specific to drive type) +smartx: Show SMART extended drive stats (specific to drive type). +temp: Show SMART drive temperature in celsius (all drives). +health: Show reported SMART status (all drives). +r_proc: Show SMART read GBytes processed over drive lifetime (SAS). +w_proc: Show SMART write GBytes processed over drive lifetime (SAS). +r_ucor: Show SMART read uncorrectable errors (SAS). +w_ucor: Show SMART write uncorrectable errors (SAS). +nonmed: Show SMART non-medium errors (SAS). +defect: Show SMART grown defect list (SAS). +hours_on: Show number of hours drive powered on (all drives). +realloc: Show SMART reallocated sectors count (ATA). +rep_ucor: Show SMART reported uncorrectable count (ATA). +cmd_to: Show SMART command timeout count (ATA). +pend_sec: Show SMART current pending sector count (ATA). +off_ucor: Show SMART offline uncorrectable errors (ATA). +ata_err: Show SMART ATA errors (ATA). +pwr_cyc: Show SMART power cycle count (ATA). +serial: Show disk serial number. +nvme_err: Show SMART NVMe errors (NVMe). +smart_test: Show SMART self-test results summary. +test_type: Show SMART self-test type (short, long... ). +test_status: Show SMART self-test status. +test_progress: Show SMART self-test percentage done. +test_ended: Show when the last SMART self-test ended (if supported). +" + +# Hack for developer testing +# +# If you set $samples to a directory containing smartctl output text files, +# we will use them instead of running smartctl on the vdevs. This can be +# useful if you want to test a bunch of different smartctl outputs. Also, if +# $samples is set, and additional 'file' column is added to the zpool output +# showing the filename. +samples= + +# get_filename_from_dir DIR +# +# Look in directory DIR and return a filename from it. The filename returned +# is chosen quasi-sequentially (based off our PID). This allows us to return +# a different filename every time this script is invoked (which we do for each +# vdev), without having to maintain state. +get_filename_from_dir() +{ + dir=$1 + pid="$$" + num_files=$(find "$dir" -maxdepth 1 -type f | wc -l) + mod=$((pid % num_files)) + i=0 + find "$dir" -type f -printf '%f\n' | while read -r file ; do + if [ "$mod" = "$i" ] ; then + echo "$file" + break + fi + i=$((i+1)) + done +} + +script="${0##*/}" + +if [ "$1" = "-h" ] ; then + echo "$helpstr" | grep "$script:" | tr -s '\t' | cut -f 2- + exit +fi + +# Sometimes, UPATH ends up /dev/(null). +# That should be corrected, but for now... +# shellcheck disable=SC2154 +if [ ! -b "$VDEV_UPATH" ]; then + somepath="${VDEV_PATH}" +else + somepath="${VDEV_UPATH}" +fi + +if [ -b "$somepath" ] && PATH="/usr/sbin:$PATH" command -v smartctl > /dev/null || [ -n "$samples" ] ; then + if [ -n "$samples" ] ; then + # cat a smartctl output text file instead of running smartctl + # on a vdev (only used for developer testing). + file=$(get_filename_from_dir "$samples") + echo "file=$file" + raw_out=$(cat "$samples/$file") + else + raw_out=$(sudo smartctl -a "$somepath") + fi + + # What kind of drive are we? Look for the right line in smartctl: + # + # SAS: + # Transport protocol: SAS + # + # SATA: + # ATA Version is: 8 + # + # NVMe: + # SMART/Health Information (NVMe Log 0xnn, NSID 0xnn) + # + out=$(echo "$raw_out" | awk ' +# SAS specific +/read:/{print "rrd="$4"\nr_cor="$5"\nr_proc="$7"\nr_ucor="$8} +/write:/{print "rwr="$4"\nw_cor="$5"\nw_proc="$7"\nw_ucor="$8} +/Non-medium error count/{print "nonmed="$4} +/Elements in grown defect list/{print "defect="$6} + +# SAS common +/SAS/{type="sas"} +/Drive Temperature:/{print "temp="$4} +# Status can be a long string, substitute spaces for '_' +/SMART Health Status:/{printf "health="; for(i=4;i<=NF-1;i++){printf "%s_", $i}; printf "%s\n", $i} +/number of hours powered up/{print "hours_on="$7; hours_on=int($7)} +/Serial number:/{print "serial="$3} + +# SATA specific +/Reallocated_Sector_Ct/{print "realloc="$10} +/Reported_Uncorrect/{print "rep_ucor="$10} +/Command_Timeout/{print "cmd_to="$10} +/Current_Pending_Sector/{print "pend_sec="$10} +/Offline_Uncorrectable/{print "off_ucor="$10} +/ATA Error Count:/{print "ata_err="$4} +/Power_Cycle_Count/{print "pwr_cyc="$10} + +# SATA common +/SATA/{type="sata"} +/Temperature_Celsius/{print "temp="$10} +/Airflow_Temperature_Cel/{print "temp="$10} +/Current Temperature:/{print "temp="$3} +/SMART overall-health self-assessment test result:/{print "health="$6} +/Power_On_Hours/{print "hours_on="$10; hours_on=int($10)} +/Serial Number:/{print "serial="$3} + +# NVMe common +/NVMe/{type="nvme"} +/Temperature:/{print "temp="$2} +/SMART overall-health self-assessment test result:/{print "health="$6} +/Power On Hours:/{gsub("[^0-9]","",$4); print "hours_on="$4} +/Serial Number:/{print "serial="$3} +/Power Cycles:/{print "pwr_cyc="$3} + +# NVMe specific +/Media and Data Integrity Errors:/{print "nvme_err="$6} + +# SMART self-test info +/Self-test execution status:/{progress=tolower($4)} # SAS +/SMART Self-test log/{test_seen=1} # SAS +/SMART Extended Self-test Log/{test_seen=1} # SATA +/# 1/{ + test_type=tolower($3"_"$4); + # Status could be one word ("Completed") or multiple ("Completed: read + # failure"). Look for the ":" to see if we need to grab more words. + + if ($5 ~ ":") + status=tolower($5""$6"_"$7) + else + status=tolower($5) + if (status=="self") + status="running"; + + if (type == "sas") { + hours=int($(NF-4)) + } else { + hours=int($(NF-1)) + # SATA reports percent remaining, rather than percent done + # Convert it to percent done. + progress=(100-int($(NF-2)))"%" + } + # When we int()-ify "hours", it converts stuff like "NOW" and "-" into + # 0. In those cases, set it to hours_on, so they will cancel out in + # the "hours_ago" calculation later on. + if (hours == 0) + hours=hours_on + + if (test_seen) { + print "test="hours_on + print "test_type="test_type + print "test_status="status + print "test_progress="progress + } + # Not all drives report hours_on + if (hours_on && hours) { + total_hours_ago=(hours_on-hours) + days_ago=int(total_hours_ago/24) + hours_ago=(total_hours_ago % 24) + if (days_ago != 0) + ago_str=days_ago"d" + if (hours_ago !=0) + ago_str=ago_str""hours_ago"h" + print "test_ended="ago_str + } +} + +END {print "type="type; ORS="\n"; print ""} +'); +fi +type=$(echo "$out" | grep '^type=' | cut -d '=' -f 2) + +# If type is not set by now, either we don't have a block device +# or smartctl failed. Either way, default to ATA and set $out to +# nothing. +if [ -z "$type" ]; then + type="sata" + out= +fi + +case $script in +smart) + # Print temperature plus common predictors of drive failure + if [ "$type" = "sas" ] ; then + scripts="temp|health|r_ucor|w_ucor" + elif [ "$type" = "sata" ] ; then + scripts="temp|health|ata_err|realloc|rep_ucor|cmd_to|pend_sec|off_ucor" + elif [ "$type" = "nvme" ] ; then + scripts="temp|health|nvme_err" + fi + ;; +smartx) + # Print some other interesting stats + if [ "$type" = "sas" ] ; then + scripts="hours_on|defect|nonmed|r_proc|w_proc" + elif [ "$type" = "sata" ] ; then + scripts="hours_on|pwr_cyc" + elif [ "$type" = "nvme" ] ; then + scripts="hours_on|pwr_cyc" + fi + ;; +smart_test) + scripts="test_type|test_status|test_progress|test_ended" + ;; +*) + scripts="$script" +esac + +with_vals=$(echo "$out" | grep -E "$scripts") +if [ -n "$with_vals" ]; then + echo "$with_vals" + without_vals=$(echo "$scripts" | tr '|' '\n' | + grep -v -E "$(echo "$with_vals" | + awk -F "=" '{print $1}')" | awk '{print $0"="}') +else + without_vals=$(echo "$scripts" | tr '|' '\n' | awk '{print $0"="}') +fi + +if [ -n "$without_vals" ]; then + echo "$without_vals" +fi diff --git a/straper/db/public/zfs/etc-zfs/zpool.d/upath b/straper/db/public/zfs/etc-zfs/zpool.d/upath new file mode 100755 index 0000000..e37ee1b --- /dev/null +++ b/straper/db/public/zfs/etc-zfs/zpool.d/upath @@ -0,0 +1,8 @@ +#!/bin/sh +if [ "$1" = "-h" ] ; then + echo "Show the underlying path for a device." + exit +fi + +# shellcheck disable=SC2154 +echo upath="$VDEV_UPATH" diff --git a/straper/db/public/zfs/etc-zfs/zpool.d/vendor b/straper/db/public/zfs/etc-zfs/zpool.d/vendor new file mode 100755 index 0000000..293effd --- /dev/null +++ b/straper/db/public/zfs/etc-zfs/zpool.d/vendor @@ -0,0 +1,92 @@ +#!/bin/sh +# +# Print some common lsblk values +# +# Any (lowercased) name symlinked to the lsblk script will be passed to lsblk +# as one of its --output names. Here's a partial list of --output names +# from the lsblk binary: +# +# Available columns (for --output): +# NAME device name +# KNAME internal kernel device name +# MAJ:MIN major:minor device number +# FSTYPE filesystem type +# MOUNTPOINT where the device is mounted +# LABEL filesystem LABEL +# UUID filesystem UUID +# RA read-ahead of the device +# RO read-only device +# RM removable device +# MODEL device identifier +# SIZE size of the device +# STATE state of the device +# OWNER user name +# GROUP group name +# MODE device node permissions +# ALIGNMENT alignment offset +# MIN-IO minimum I/O size +# OPT-IO optimal I/O size +# PHY-SEC physical sector size +# LOG-SEC logical sector size +# ROTA rotational device +# SCHED I/O scheduler name +# RQ-SIZE request queue size +# TYPE device type +# DISC-ALN discard alignment offset +# DISC-GRAN discard granularity +# DISC-MAX discard max bytes +# DISC-ZERO discard zeroes data +# +# If the script is run as just 'lsblk' then print out disk size, vendor, +# and model number. + + +helpstr=" +label: Show filesystem label. +model: Show disk model number. +size: Show the disk capacity. +vendor: Show the disk vendor. +lsblk: Show the disk size, vendor, and model number." + +script="${0##*/}" + +if [ "$1" = "-h" ] ; then + echo "$helpstr" | grep "$script:" | tr -s '\t' | cut -f 2- + exit +fi + +if [ "$script" = "lsblk" ] ; then + list="size vendor model" +else + list=$(echo "$script" | tr '[:upper:]' '[:lower:]') +fi + +# Sometimes, UPATH ends up /dev/(null). +# That should be corrected, but for now... +# shellcheck disable=SC2154 +if [ ! -b "$VDEV_UPATH" ]; then + somepath="${VDEV_PATH}" +else + somepath="${VDEV_UPATH}" +fi + +# Older versions of lsblk don't support all these values (like SERIAL). +for i in $list ; do + + # Special case: Looking up the size of a file-based vdev can't + # be done with lsblk. + if [ "$i" = "size" ] && [ -f "$somepath" ] ; then + size=$(du -h --apparent-size "$somepath" | cut -f 1) + echo "size=$size" + continue + fi + + + val="" + if val=$(eval "lsblk -dl -n -o $i $somepath 2>/dev/null") ; then + # Remove leading/trailing whitespace from value + val=$(echo "$val" | sed -e 's/^[[:space:]]*//' \ + -e 's/[[:space:]]*$//') + fi + echo "$i=$val" +done diff --git a/straper/db/public/zfs/etc-zfs/zpool.d/w_proc b/straper/db/public/zfs/etc-zfs/zpool.d/w_proc new file mode 100755 index 0000000..8ad3e10 --- /dev/null +++ b/straper/db/public/zfs/etc-zfs/zpool.d/w_proc @@ -0,0 +1,249 @@ +#!/bin/sh +# +# Show SMART stats +# + +helpstr=" +smart: Show SMART temperature and error stats (specific to drive type) +smartx: Show SMART extended drive stats (specific to drive type). +temp: Show SMART drive temperature in celsius (all drives). +health: Show reported SMART status (all drives). +r_proc: Show SMART read GBytes processed over drive lifetime (SAS). +w_proc: Show SMART write GBytes processed over drive lifetime (SAS). +r_ucor: Show SMART read uncorrectable errors (SAS). +w_ucor: Show SMART write uncorrectable errors (SAS). +nonmed: Show SMART non-medium errors (SAS). +defect: Show SMART grown defect list (SAS). +hours_on: Show number of hours drive powered on (all drives). +realloc: Show SMART reallocated sectors count (ATA). +rep_ucor: Show SMART reported uncorrectable count (ATA). +cmd_to: Show SMART command timeout count (ATA). +pend_sec: Show SMART current pending sector count (ATA). +off_ucor: Show SMART offline uncorrectable errors (ATA). +ata_err: Show SMART ATA errors (ATA). +pwr_cyc: Show SMART power cycle count (ATA). +serial: Show disk serial number. +nvme_err: Show SMART NVMe errors (NVMe). +smart_test: Show SMART self-test results summary. +test_type: Show SMART self-test type (short, long... ). +test_status: Show SMART self-test status. +test_progress: Show SMART self-test percentage done. +test_ended: Show when the last SMART self-test ended (if supported). +" + +# Hack for developer testing +# +# If you set $samples to a directory containing smartctl output text files, +# we will use them instead of running smartctl on the vdevs. This can be +# useful if you want to test a bunch of different smartctl outputs. Also, if +# $samples is set, and additional 'file' column is added to the zpool output +# showing the filename. +samples= + +# get_filename_from_dir DIR +# +# Look in directory DIR and return a filename from it. The filename returned +# is chosen quasi-sequentially (based off our PID). This allows us to return +# a different filename every time this script is invoked (which we do for each +# vdev), without having to maintain state. +get_filename_from_dir() +{ + dir=$1 + pid="$$" + num_files=$(find "$dir" -maxdepth 1 -type f | wc -l) + mod=$((pid % num_files)) + i=0 + find "$dir" -type f -printf '%f\n' | while read -r file ; do + if [ "$mod" = "$i" ] ; then + echo "$file" + break + fi + i=$((i+1)) + done +} + +script="${0##*/}" + +if [ "$1" = "-h" ] ; then + echo "$helpstr" | grep "$script:" | tr -s '\t' | cut -f 2- + exit +fi + +# Sometimes, UPATH ends up /dev/(null). +# That should be corrected, but for now... +# shellcheck disable=SC2154 +if [ ! -b "$VDEV_UPATH" ]; then + somepath="${VDEV_PATH}" +else + somepath="${VDEV_UPATH}" +fi + +if [ -b "$somepath" ] && PATH="/usr/sbin:$PATH" command -v smartctl > /dev/null || [ -n "$samples" ] ; then + if [ -n "$samples" ] ; then + # cat a smartctl output text file instead of running smartctl + # on a vdev (only used for developer testing). + file=$(get_filename_from_dir "$samples") + echo "file=$file" + raw_out=$(cat "$samples/$file") + else + raw_out=$(sudo smartctl -a "$somepath") + fi + + # What kind of drive are we? Look for the right line in smartctl: + # + # SAS: + # Transport protocol: SAS + # + # SATA: + # ATA Version is: 8 + # + # NVMe: + # SMART/Health Information (NVMe Log 0xnn, NSID 0xnn) + # + out=$(echo "$raw_out" | awk ' +# SAS specific +/read:/{print "rrd="$4"\nr_cor="$5"\nr_proc="$7"\nr_ucor="$8} +/write:/{print "rwr="$4"\nw_cor="$5"\nw_proc="$7"\nw_ucor="$8} +/Non-medium error count/{print "nonmed="$4} +/Elements in grown defect list/{print "defect="$6} + +# SAS common +/SAS/{type="sas"} +/Drive Temperature:/{print "temp="$4} +# Status can be a long string, substitute spaces for '_' +/SMART Health Status:/{printf "health="; for(i=4;i<=NF-1;i++){printf "%s_", $i}; printf "%s\n", $i} +/number of hours powered up/{print "hours_on="$7; hours_on=int($7)} +/Serial number:/{print "serial="$3} + +# SATA specific +/Reallocated_Sector_Ct/{print "realloc="$10} +/Reported_Uncorrect/{print "rep_ucor="$10} +/Command_Timeout/{print "cmd_to="$10} +/Current_Pending_Sector/{print "pend_sec="$10} +/Offline_Uncorrectable/{print "off_ucor="$10} +/ATA Error Count:/{print "ata_err="$4} +/Power_Cycle_Count/{print "pwr_cyc="$10} + +# SATA common +/SATA/{type="sata"} +/Temperature_Celsius/{print "temp="$10} +/Airflow_Temperature_Cel/{print "temp="$10} +/Current Temperature:/{print "temp="$3} +/SMART overall-health self-assessment test result:/{print "health="$6} +/Power_On_Hours/{print "hours_on="$10; hours_on=int($10)} +/Serial Number:/{print "serial="$3} + +# NVMe common +/NVMe/{type="nvme"} +/Temperature:/{print "temp="$2} +/SMART overall-health self-assessment test result:/{print "health="$6} +/Power On Hours:/{gsub("[^0-9]","",$4); print "hours_on="$4} +/Serial Number:/{print "serial="$3} +/Power Cycles:/{print "pwr_cyc="$3} + +# NVMe specific +/Media and Data Integrity Errors:/{print "nvme_err="$6} + +# SMART self-test info +/Self-test execution status:/{progress=tolower($4)} # SAS +/SMART Self-test log/{test_seen=1} # SAS +/SMART Extended Self-test Log/{test_seen=1} # SATA +/# 1/{ + test_type=tolower($3"_"$4); + # Status could be one word ("Completed") or multiple ("Completed: read + # failure"). Look for the ":" to see if we need to grab more words. + + if ($5 ~ ":") + status=tolower($5""$6"_"$7) + else + status=tolower($5) + if (status=="self") + status="running"; + + if (type == "sas") { + hours=int($(NF-4)) + } else { + hours=int($(NF-1)) + # SATA reports percent remaining, rather than percent done + # Convert it to percent done. + progress=(100-int($(NF-2)))"%" + } + # When we int()-ify "hours", it converts stuff like "NOW" and "-" into + # 0. In those cases, set it to hours_on, so they will cancel out in + # the "hours_ago" calculation later on. + if (hours == 0) + hours=hours_on + + if (test_seen) { + print "test="hours_on + print "test_type="test_type + print "test_status="status + print "test_progress="progress + } + # Not all drives report hours_on + if (hours_on && hours) { + total_hours_ago=(hours_on-hours) + days_ago=int(total_hours_ago/24) + hours_ago=(total_hours_ago % 24) + if (days_ago != 0) + ago_str=days_ago"d" + if (hours_ago !=0) + ago_str=ago_str""hours_ago"h" + print "test_ended="ago_str + } +} + +END {print "type="type; ORS="\n"; print ""} +'); +fi +type=$(echo "$out" | grep '^type=' | cut -d '=' -f 2) + +# If type is not set by now, either we don't have a block device +# or smartctl failed. Either way, default to ATA and set $out to +# nothing. +if [ -z "$type" ]; then + type="sata" + out= +fi + +case $script in +smart) + # Print temperature plus common predictors of drive failure + if [ "$type" = "sas" ] ; then + scripts="temp|health|r_ucor|w_ucor" + elif [ "$type" = "sata" ] ; then + scripts="temp|health|ata_err|realloc|rep_ucor|cmd_to|pend_sec|off_ucor" + elif [ "$type" = "nvme" ] ; then + scripts="temp|health|nvme_err" + fi + ;; +smartx) + # Print some other interesting stats + if [ "$type" = "sas" ] ; then + scripts="hours_on|defect|nonmed|r_proc|w_proc" + elif [ "$type" = "sata" ] ; then + scripts="hours_on|pwr_cyc" + elif [ "$type" = "nvme" ] ; then + scripts="hours_on|pwr_cyc" + fi + ;; +smart_test) + scripts="test_type|test_status|test_progress|test_ended" + ;; +*) + scripts="$script" +esac + +with_vals=$(echo "$out" | grep -E "$scripts") +if [ -n "$with_vals" ]; then + echo "$with_vals" + without_vals=$(echo "$scripts" | tr '|' '\n' | + grep -v -E "$(echo "$with_vals" | + awk -F "=" '{print $1}')" | awk '{print $0"="}') +else + without_vals=$(echo "$scripts" | tr '|' '\n' | awk '{print $0"="}') +fi + +if [ -n "$without_vals" ]; then + echo "$without_vals" +fi diff --git a/straper/db/public/zfs/etc-zfs/zpool.d/w_ucor b/straper/db/public/zfs/etc-zfs/zpool.d/w_ucor new file mode 100755 index 0000000..8ad3e10 --- /dev/null +++ b/straper/db/public/zfs/etc-zfs/zpool.d/w_ucor @@ -0,0 +1,249 @@ +#!/bin/sh +# +# Show SMART stats +# + +helpstr=" +smart: Show SMART temperature and error stats (specific to drive type) +smartx: Show SMART extended drive stats (specific to drive type). +temp: Show SMART drive temperature in celsius (all drives). +health: Show reported SMART status (all drives). +r_proc: Show SMART read GBytes processed over drive lifetime (SAS). +w_proc: Show SMART write GBytes processed over drive lifetime (SAS). +r_ucor: Show SMART read uncorrectable errors (SAS). +w_ucor: Show SMART write uncorrectable errors (SAS). +nonmed: Show SMART non-medium errors (SAS). +defect: Show SMART grown defect list (SAS). +hours_on: Show number of hours drive powered on (all drives). +realloc: Show SMART reallocated sectors count (ATA). +rep_ucor: Show SMART reported uncorrectable count (ATA). +cmd_to: Show SMART command timeout count (ATA). +pend_sec: Show SMART current pending sector count (ATA). +off_ucor: Show SMART offline uncorrectable errors (ATA). +ata_err: Show SMART ATA errors (ATA). +pwr_cyc: Show SMART power cycle count (ATA). +serial: Show disk serial number. +nvme_err: Show SMART NVMe errors (NVMe). +smart_test: Show SMART self-test results summary. +test_type: Show SMART self-test type (short, long... ). +test_status: Show SMART self-test status. +test_progress: Show SMART self-test percentage done. +test_ended: Show when the last SMART self-test ended (if supported). +" + +# Hack for developer testing +# +# If you set $samples to a directory containing smartctl output text files, +# we will use them instead of running smartctl on the vdevs. This can be +# useful if you want to test a bunch of different smartctl outputs. Also, if +# $samples is set, and additional 'file' column is added to the zpool output +# showing the filename. +samples= + +# get_filename_from_dir DIR +# +# Look in directory DIR and return a filename from it. The filename returned +# is chosen quasi-sequentially (based off our PID). This allows us to return +# a different filename every time this script is invoked (which we do for each +# vdev), without having to maintain state. +get_filename_from_dir() +{ + dir=$1 + pid="$$" + num_files=$(find "$dir" -maxdepth 1 -type f | wc -l) + mod=$((pid % num_files)) + i=0 + find "$dir" -type f -printf '%f\n' | while read -r file ; do + if [ "$mod" = "$i" ] ; then + echo "$file" + break + fi + i=$((i+1)) + done +} + +script="${0##*/}" + +if [ "$1" = "-h" ] ; then + echo "$helpstr" | grep "$script:" | tr -s '\t' | cut -f 2- + exit +fi + +# Sometimes, UPATH ends up /dev/(null). +# That should be corrected, but for now... +# shellcheck disable=SC2154 +if [ ! -b "$VDEV_UPATH" ]; then + somepath="${VDEV_PATH}" +else + somepath="${VDEV_UPATH}" +fi + +if [ -b "$somepath" ] && PATH="/usr/sbin:$PATH" command -v smartctl > /dev/null || [ -n "$samples" ] ; then + if [ -n "$samples" ] ; then + # cat a smartctl output text file instead of running smartctl + # on a vdev (only used for developer testing). + file=$(get_filename_from_dir "$samples") + echo "file=$file" + raw_out=$(cat "$samples/$file") + else + raw_out=$(sudo smartctl -a "$somepath") + fi + + # What kind of drive are we? Look for the right line in smartctl: + # + # SAS: + # Transport protocol: SAS + # + # SATA: + # ATA Version is: 8 + # + # NVMe: + # SMART/Health Information (NVMe Log 0xnn, NSID 0xnn) + # + out=$(echo "$raw_out" | awk ' +# SAS specific +/read:/{print "rrd="$4"\nr_cor="$5"\nr_proc="$7"\nr_ucor="$8} +/write:/{print "rwr="$4"\nw_cor="$5"\nw_proc="$7"\nw_ucor="$8} +/Non-medium error count/{print "nonmed="$4} +/Elements in grown defect list/{print "defect="$6} + +# SAS common +/SAS/{type="sas"} +/Drive Temperature:/{print "temp="$4} +# Status can be a long string, substitute spaces for '_' +/SMART Health Status:/{printf "health="; for(i=4;i<=NF-1;i++){printf "%s_", $i}; printf "%s\n", $i} +/number of hours powered up/{print "hours_on="$7; hours_on=int($7)} +/Serial number:/{print "serial="$3} + +# SATA specific +/Reallocated_Sector_Ct/{print "realloc="$10} +/Reported_Uncorrect/{print "rep_ucor="$10} +/Command_Timeout/{print "cmd_to="$10} +/Current_Pending_Sector/{print "pend_sec="$10} +/Offline_Uncorrectable/{print "off_ucor="$10} +/ATA Error Count:/{print "ata_err="$4} +/Power_Cycle_Count/{print "pwr_cyc="$10} + +# SATA common +/SATA/{type="sata"} +/Temperature_Celsius/{print "temp="$10} +/Airflow_Temperature_Cel/{print "temp="$10} +/Current Temperature:/{print "temp="$3} +/SMART overall-health self-assessment test result:/{print "health="$6} +/Power_On_Hours/{print "hours_on="$10; hours_on=int($10)} +/Serial Number:/{print "serial="$3} + +# NVMe common +/NVMe/{type="nvme"} +/Temperature:/{print "temp="$2} +/SMART overall-health self-assessment test result:/{print "health="$6} +/Power On Hours:/{gsub("[^0-9]","",$4); print "hours_on="$4} +/Serial Number:/{print "serial="$3} +/Power Cycles:/{print "pwr_cyc="$3} + +# NVMe specific +/Media and Data Integrity Errors:/{print "nvme_err="$6} + +# SMART self-test info +/Self-test execution status:/{progress=tolower($4)} # SAS +/SMART Self-test log/{test_seen=1} # SAS +/SMART Extended Self-test Log/{test_seen=1} # SATA +/# 1/{ + test_type=tolower($3"_"$4); + # Status could be one word ("Completed") or multiple ("Completed: read + # failure"). Look for the ":" to see if we need to grab more words. + + if ($5 ~ ":") + status=tolower($5""$6"_"$7) + else + status=tolower($5) + if (status=="self") + status="running"; + + if (type == "sas") { + hours=int($(NF-4)) + } else { + hours=int($(NF-1)) + # SATA reports percent remaining, rather than percent done + # Convert it to percent done. + progress=(100-int($(NF-2)))"%" + } + # When we int()-ify "hours", it converts stuff like "NOW" and "-" into + # 0. In those cases, set it to hours_on, so they will cancel out in + # the "hours_ago" calculation later on. + if (hours == 0) + hours=hours_on + + if (test_seen) { + print "test="hours_on + print "test_type="test_type + print "test_status="status + print "test_progress="progress + } + # Not all drives report hours_on + if (hours_on && hours) { + total_hours_ago=(hours_on-hours) + days_ago=int(total_hours_ago/24) + hours_ago=(total_hours_ago % 24) + if (days_ago != 0) + ago_str=days_ago"d" + if (hours_ago !=0) + ago_str=ago_str""hours_ago"h" + print "test_ended="ago_str + } +} + +END {print "type="type; ORS="\n"; print ""} +'); +fi +type=$(echo "$out" | grep '^type=' | cut -d '=' -f 2) + +# If type is not set by now, either we don't have a block device +# or smartctl failed. Either way, default to ATA and set $out to +# nothing. +if [ -z "$type" ]; then + type="sata" + out= +fi + +case $script in +smart) + # Print temperature plus common predictors of drive failure + if [ "$type" = "sas" ] ; then + scripts="temp|health|r_ucor|w_ucor" + elif [ "$type" = "sata" ] ; then + scripts="temp|health|ata_err|realloc|rep_ucor|cmd_to|pend_sec|off_ucor" + elif [ "$type" = "nvme" ] ; then + scripts="temp|health|nvme_err" + fi + ;; +smartx) + # Print some other interesting stats + if [ "$type" = "sas" ] ; then + scripts="hours_on|defect|nonmed|r_proc|w_proc" + elif [ "$type" = "sata" ] ; then + scripts="hours_on|pwr_cyc" + elif [ "$type" = "nvme" ] ; then + scripts="hours_on|pwr_cyc" + fi + ;; +smart_test) + scripts="test_type|test_status|test_progress|test_ended" + ;; +*) + scripts="$script" +esac + +with_vals=$(echo "$out" | grep -E "$scripts") +if [ -n "$with_vals" ]; then + echo "$with_vals" + without_vals=$(echo "$scripts" | tr '|' '\n' | + grep -v -E "$(echo "$with_vals" | + awk -F "=" '{print $1}')" | awk '{print $0"="}') +else + without_vals=$(echo "$scripts" | tr '|' '\n' | awk '{print $0"="}') +fi + +if [ -n "$without_vals" ]; then + echo "$without_vals" +fi |