diff options
Diffstat (limited to 'straper/db/public/zfs/etc-zfs/zed.d')
15 files changed, 2524 insertions, 0 deletions
diff --git a/straper/db/public/zfs/etc-zfs/zed.d/all-syslog.sh b/straper/db/public/zfs/etc-zfs/zed.d/all-syslog.sh new file mode 100755 index 0000000..5f60114 --- /dev/null +++ b/straper/db/public/zfs/etc-zfs/zed.d/all-syslog.sh @@ -0,0 +1,52 @@ +#!/bin/sh +# shellcheck disable=SC2154 +# +# Copyright (C) 2013-2014 Lawrence Livermore National Security, LLC. +# Copyright (c) 2020 by Delphix. All rights reserved. +# + +# +# Log the zevent via syslog. +# + +[ -f "${ZED_ZEDLET_DIR}/zed.rc" ] && . "${ZED_ZEDLET_DIR}/zed.rc" +. "${ZED_ZEDLET_DIR}/zed-functions.sh" + +zed_exit_if_ignoring_this_event + +# build a string of name=value pairs for this event +msg="eid=${ZEVENT_EID} class=${ZEVENT_SUBCLASS}" + +if [ "${ZED_SYSLOG_DISPLAY_GUIDS}" = "1" ]; then + [ -n "${ZEVENT_POOL_GUID}" ] && msg="${msg} pool_guid=${ZEVENT_POOL_GUID}" + [ -n "${ZEVENT_VDEV_GUID}" ] && msg="${msg} vdev_guid=${ZEVENT_VDEV_GUID}" +else + [ -n "${ZEVENT_POOL}" ] && msg="${msg} pool='${ZEVENT_POOL}'" + [ -n "${ZEVENT_VDEV_PATH}" ] && msg="${msg} vdev=${ZEVENT_VDEV_PATH##*/}" +fi + +# log pool state if state is anything other than 'ACTIVE' +[ -n "${ZEVENT_POOL_STATE_STR}" ] && [ "$ZEVENT_POOL_STATE" -ne 0 ] && \ + msg="${msg} pool_state=${ZEVENT_POOL_STATE_STR}" + +# Log the following payload nvpairs if they are present +[ -n "${ZEVENT_VDEV_STATE_STR}" ] && msg="${msg} vdev_state=${ZEVENT_VDEV_STATE_STR}" +[ -n "${ZEVENT_CKSUM_ALGORITHM}" ] && msg="${msg} algorithm=${ZEVENT_CKSUM_ALGORITHM}" +[ -n "${ZEVENT_ZIO_SIZE}" ] && msg="${msg} size=${ZEVENT_ZIO_SIZE}" +[ -n "${ZEVENT_ZIO_OFFSET}" ] && msg="${msg} offset=${ZEVENT_ZIO_OFFSET}" +[ -n "${ZEVENT_ZIO_PRIORITY}" ] && msg="${msg} priority=${ZEVENT_ZIO_PRIORITY}" +[ -n "${ZEVENT_ZIO_ERR}" ] && msg="${msg} err=${ZEVENT_ZIO_ERR}" +[ -n "${ZEVENT_ZIO_FLAGS}" ] && msg="${msg} flags=$(printf '0x%x' "${ZEVENT_ZIO_FLAGS}")" + +# log delays that are >= 10 milisec +[ -n "${ZEVENT_ZIO_DELAY}" ] && [ "$ZEVENT_ZIO_DELAY" -gt 10000000 ] && \ + msg="${msg} delay=$((ZEVENT_ZIO_DELAY / 1000000))ms" + +# list the bookmark data together +# shellcheck disable=SC2153 +[ -n "${ZEVENT_ZIO_OBJSET}" ] && \ + msg="${msg} bookmark=${ZEVENT_ZIO_OBJSET}:${ZEVENT_ZIO_OBJECT}:${ZEVENT_ZIO_LEVEL}:${ZEVENT_ZIO_BLKID}" + +zed_log_msg "${msg}" + +exit 0 diff --git a/straper/db/public/zfs/etc-zfs/zed.d/data-notify.sh b/straper/db/public/zfs/etc-zfs/zed.d/data-notify.sh new file mode 100755 index 0000000..9846769 --- /dev/null +++ b/straper/db/public/zfs/etc-zfs/zed.d/data-notify.sh @@ -0,0 +1,44 @@ +#!/bin/sh +# shellcheck disable=SC2154 +# +# Send notification in response to a DATA error. +# +# Only one notification per ZED_NOTIFY_INTERVAL_SECS will be sent for a given +# class/pool/[vdev] combination. This protects against spamming the recipient +# should multiple events occur together in time for the same pool/[vdev]. +# +# Exit codes: +# 0: notification sent +# 1: notification failed +# 2: notification not configured +# 3: notification suppressed +# 9: internal error + +[ -f "${ZED_ZEDLET_DIR}/zed.rc" ] && . "${ZED_ZEDLET_DIR}/zed.rc" +. "${ZED_ZEDLET_DIR}/zed-functions.sh" + +[ -n "${ZEVENT_POOL}" ] || exit 9 +[ -n "${ZEVENT_SUBCLASS}" ] || exit 9 +[ -n "${ZED_NOTIFY_DATA}" ] || exit 3 + +rate_limit_tag="${ZEVENT_POOL};${ZEVENT_VDEV_GUID:-0};${ZEVENT_SUBCLASS};notify" +zed_rate_limit "${rate_limit_tag}" || exit 3 + +umask 077 +note_subject="ZFS ${ZEVENT_SUBCLASS} error for ${ZEVENT_POOL} on $(hostname)" +note_pathname="$(mktemp)" +{ + echo "ZFS has detected a data error:" + echo + echo " eid: ${ZEVENT_EID}" + echo " class: ${ZEVENT_SUBCLASS}" + echo " host: $(hostname)" + echo " time: ${ZEVENT_TIME_STRING}" + echo " error: ${ZEVENT_ZIO_ERR}" + echo " objid: ${ZEVENT_ZIO_OBJSET}:${ZEVENT_ZIO_OBJECT}" + echo " pool: ${ZEVENT_POOL}" +} > "${note_pathname}" + +zed_notify "${note_subject}" "${note_pathname}"; rv=$? +rm -f "${note_pathname}" +exit "${rv}" diff --git a/straper/db/public/zfs/etc-zfs/zed.d/deadman-slot_off.sh b/straper/db/public/zfs/etc-zfs/zed.d/deadman-slot_off.sh new file mode 100755 index 0000000..7b339b3 --- /dev/null +++ b/straper/db/public/zfs/etc-zfs/zed.d/deadman-slot_off.sh @@ -0,0 +1,71 @@ +#!/bin/sh +# shellcheck disable=SC3014,SC2154,SC2086,SC2034 +# +# Turn off disk's enclosure slot if an I/O is hung triggering the deadman. +# +# It's possible for outstanding I/O to a misbehaving SCSI disk to neither +# promptly complete or return an error. This can occur due to retry and +# recovery actions taken by the SCSI layer, driver, or disk. When it occurs +# the pool will be unresponsive even though there may be sufficient redundancy +# configured to proceeded without this single disk. +# +# When a hung I/O is detected by the kmods it will be posted as a deadman +# event. By default an I/O is considered to be hung after 5 minutes. This +# value can be changed with the zfs_deadman_ziotime_ms module parameter. +# If ZED_POWER_OFF_ENCLOSURE_SLOT_ON_DEADMAN is set the disk's enclosure +# slot will be powered off causing the outstanding I/O to fail. The ZED +# will then handle this like a normal disk failure and FAULT the vdev. +# +# We assume the user will be responsible for turning the slot back on +# after replacing the disk. +# +# Note that this script requires that your enclosure be supported by the +# Linux SCSI Enclosure services (SES) driver. The script will do nothing +# if you have no enclosure, or if your enclosure isn't supported. +# +# Exit codes: +# 0: slot successfully powered off +# 1: enclosure not available +# 2: ZED_POWER_OFF_ENCLOSURE_SLOT_ON_DEADMAN disabled +# 3: System not configured to wait on deadman +# 4: The enclosure sysfs path passed from ZFS does not exist +# 5: Enclosure slot didn't actually turn off after we told it to + +[ -f "${ZED_ZEDLET_DIR}/zed.rc" ] && . "${ZED_ZEDLET_DIR}/zed.rc" +. "${ZED_ZEDLET_DIR}/zed-functions.sh" + +if [ ! -d /sys/class/enclosure ] ; then + # No JBOD enclosure or NVMe slots + exit 1 +fi + +if [ "${ZED_POWER_OFF_ENCLOSURE_SLOT_ON_DEADMAN}" != "1" ] ; then + exit 2 +fi + +if [ "$ZEVENT_POOL_FAILMODE" != "wait" ] ; then + exit 3 +fi + +if [ ! -f "$ZEVENT_VDEV_ENC_SYSFS_PATH/power_status" ] ; then + exit 4 +fi + +# Turn off the slot and wait for sysfs to report that the slot is off. +# It can take ~400ms on some enclosures and multiple retries may be needed. +for i in $(seq 1 20) ; do + echo "off" | tee "$ZEVENT_VDEV_ENC_SYSFS_PATH/power_status" + + for j in $(seq 1 5) ; do + if [ "$(cat $ZEVENT_VDEV_ENC_SYSFS_PATH/power_status)" == "off" ] ; then + break 2 + fi + sleep 0.1 + done +done + +if [ "$(cat $ZEVENT_VDEV_ENC_SYSFS_PATH/power_status)" != "off" ] ; then + exit 5 +fi + +zed_log_msg "powered down slot $ZEVENT_VDEV_ENC_SYSFS_PATH for $ZEVENT_VDEV_PATH" diff --git a/straper/db/public/zfs/etc-zfs/zed.d/history_event-zfs-list-cacher.sh b/straper/db/public/zfs/etc-zfs/zed.d/history_event-zfs-list-cacher.sh new file mode 100755 index 0000000..cacf921 --- /dev/null +++ b/straper/db/public/zfs/etc-zfs/zed.d/history_event-zfs-list-cacher.sh @@ -0,0 +1,85 @@ +#!/bin/sh +# shellcheck disable=SC2154 +# +# Track changes to enumerated pools for use in early-boot +set -ef + +FSLIST="/etc/zfs/zfs-list.cache/${ZEVENT_POOL}" +FSLIST_TMP="/run/zfs-list.cache@${ZEVENT_POOL}" + +# If the pool specific cache file is not writeable, abort +[ -w "${FSLIST}" ] || exit 0 + +[ -f "${ZED_ZEDLET_DIR}/zed.rc" ] && . "${ZED_ZEDLET_DIR}/zed.rc" +. "${ZED_ZEDLET_DIR}/zed-functions.sh" + +[ "$ZEVENT_SUBCLASS" != "history_event" ] && exit 0 +zed_check_cmd "${ZFS}" sort diff + +# If we are acting on a snapshot, we have nothing to do +[ "${ZEVENT_HISTORY_DSNAME%@*}" = "${ZEVENT_HISTORY_DSNAME}" ] || exit 0 + +# We lock the output file to avoid simultaneous writes. +# If we run into trouble, log and drop the lock +abort_alter() { + zed_log_msg "Error updating zfs-list.cache for ${ZEVENT_POOL}!" + zed_unlock "${FSLIST}" +} + +finished() { + zed_unlock "${FSLIST}" + trap - EXIT + exit 0 +} + +case "${ZEVENT_HISTORY_INTERNAL_NAME}" in + create|"finish receiving"|import|destroy|rename) + ;; + + export) + zed_lock "${FSLIST}" + trap abort_alter EXIT + echo > "${FSLIST}" + finished + ;; + + set|inherit) + # Only act if one of the tracked properties is altered. + case "${ZEVENT_HISTORY_INTERNAL_STR%%=*}" in + canmount|mountpoint|atime|relatime|devices|exec|readonly| \ + setuid|nbmand|encroot|keylocation|org.openzfs.systemd:requires| \ + org.openzfs.systemd:requires-mounts-for| \ + org.openzfs.systemd:before|org.openzfs.systemd:after| \ + org.openzfs.systemd:wanted-by|org.openzfs.systemd:required-by| \ + org.openzfs.systemd:nofail|org.openzfs.systemd:ignore \ + ) ;; + *) exit 0 ;; + esac + ;; + + *) + # Ignore all other events. + exit 0 + ;; +esac + +zed_lock "${FSLIST}" +trap abort_alter EXIT + +PROPS="name,mountpoint,canmount,atime,relatime,devices,exec\ +,readonly,setuid,nbmand,encroot,keylocation\ +,org.openzfs.systemd:requires,org.openzfs.systemd:requires-mounts-for\ +,org.openzfs.systemd:before,org.openzfs.systemd:after\ +,org.openzfs.systemd:wanted-by,org.openzfs.systemd:required-by\ +,org.openzfs.systemd:nofail,org.openzfs.systemd:ignore" + +"${ZFS}" list -H -t filesystem -o "${PROPS}" -r "${ZEVENT_POOL}" > "${FSLIST_TMP}" + +# Sort the output so that it is stable +sort "${FSLIST_TMP}" -o "${FSLIST_TMP}" + +# Don't modify the file if it hasn't changed +diff -q "${FSLIST_TMP}" "${FSLIST}" || cat "${FSLIST_TMP}" > "${FSLIST}" +rm -f "${FSLIST_TMP}" + +finished diff --git a/straper/db/public/zfs/etc-zfs/zed.d/pool_import-led.sh b/straper/db/public/zfs/etc-zfs/zed.d/pool_import-led.sh new file mode 100755 index 0000000..40cb61f --- /dev/null +++ b/straper/db/public/zfs/etc-zfs/zed.d/pool_import-led.sh @@ -0,0 +1,242 @@ +#!/bin/sh +# shellcheck disable=SC2154 +# +# Turn off/on vdevs' enclosure fault LEDs when their pool's state changes. +# +# Turn a vdev's fault LED on if it becomes FAULTED, DEGRADED or UNAVAIL. +# Turn its LED off when it's back ONLINE again. +# +# This script run in two basic modes: +# +# 1. If $ZEVENT_VDEV_ENC_SYSFS_PATH and $ZEVENT_VDEV_STATE_STR are set, then +# only set the LED for that particular vdev. This is the case for statechange +# events and some vdev_* events. +# +# 2. If those vars are not set, then check the state of all vdevs in the pool +# and set the LEDs accordingly. This is the case for pool_import events. +# +# Note that this script requires that your enclosure be supported by the +# Linux SCSI Enclosure services (SES) driver. The script will do nothing +# if you have no enclosure, or if your enclosure isn't supported. +# +# Exit codes: +# 0: enclosure led successfully set +# 1: enclosure leds not available +# 2: enclosure leds administratively disabled +# 3: The led sysfs path passed from ZFS does not exist +# 4: $ZPOOL not set +# 5: awk is not installed + +[ -f "${ZED_ZEDLET_DIR}/zed.rc" ] && . "${ZED_ZEDLET_DIR}/zed.rc" +. "${ZED_ZEDLET_DIR}/zed-functions.sh" + +if [ ! -d /sys/class/enclosure ] && [ ! -d /sys/bus/pci/slots ] ; then + # No JBOD enclosure or NVMe slots + exit 1 +fi + +if [ "${ZED_USE_ENCLOSURE_LEDS}" != "1" ] ; then + exit 2 +fi + +zed_check_cmd "$ZPOOL" || exit 4 +zed_check_cmd awk || exit 5 + +# Global used in set_led debug print +vdev="" + +# check_and_set_led (file, val) +# +# Read an enclosure sysfs file, and write it if it's not already set to 'val' +# +# Arguments +# file: sysfs file to set (like /sys/class/enclosure/0:0:1:0/SLOT 10/fault) +# val: value to set it to +# +# Return +# 0 on success, 3 on missing sysfs path +# +check_and_set_led() +{ + file="$1" + val="$2" + + if [ -z "$val" ]; then + return 0 + fi + + if [ ! -e "$file" ] ; then + return 3 + fi + + # If another process is accessing the LED when we attempt to update it, + # the update will be lost so retry until the LED actually changes or we + # timeout. + for _ in 1 2 3 4 5; do + # We want to check the current state first, since writing to the + # 'fault' entry always causes a SES command, even if the + # current state is already what you want. + read -r current < "${file}" + + # On some enclosures if you write 1 to fault, and read it back, + # it will return 2. Treat all non-zero values as 1 for + # simplicity. + if [ "$current" != "0" ] ; then + current=1 + fi + + if [ "$current" != "$val" ] ; then + echo "$val" > "$file" + zed_log_msg "vdev $vdev set '$file' LED to $val" + else + break + fi + done +} + +# Fault LEDs for JBODs and NVMe drives are handled a little differently. +# +# On JBODs the fault LED is called 'fault' and on a path like this: +# +# /sys/class/enclosure/0:0:1:0/SLOT 10/fault +# +# On NVMe it's called 'attention' and on a path like this: +# +# /sys/bus/pci/slot/0/attention +# +# This function returns the full path to the fault LED file for a given +# enclosure/slot directory. +# +path_to_led() +{ + dir=$1 + if [ -f "$dir/fault" ] ; then + echo "$dir/fault" + elif [ -f "$dir/attention" ] ; then + echo "$dir/attention" + fi +} + +state_to_val() +{ + state="$1" + case "$state" in + FAULTED|DEGRADED|UNAVAIL|REMOVED) + echo 1 + ;; + ONLINE) + echo 0 + ;; + *) + echo "invalid state: $state" + ;; + esac +} + +# +# Given a nvme name like 'nvme0n1', pass back its slot directory +# like "/sys/bus/pci/slots/0" +# +nvme_dev_to_slot() +{ + dev="$1" + + # Get the address "0000:01:00.0" + read -r address < "/sys/class/block/$dev/device/address" + + find /sys/bus/pci/slots -regex '.*/[0-9]+/address$' | \ + while read -r sys_addr; do + read -r this_address < "$sys_addr" + + # The format of address is a little different between + # /sys/class/block/$dev/device/address and + # /sys/bus/pci/slots/ + # + # address= "0000:01:00.0" + # this_address = "0000:01:00" + # + if echo "$address" | grep -Eq ^"$this_address" ; then + echo "${sys_addr%/*}" + break + fi + done +} + + +# process_pool (pool) +# +# Iterate through a pool and set the vdevs' enclosure slot LEDs to +# those vdevs' state. +# +# Arguments +# pool: Pool name. +# +# Return +# 0 on success, 3 on missing sysfs path +# +process_pool() +{ + pool="$1" + + # The output will be the vdevs only (from "grep '/dev/'"): + # + # U45 ONLINE 0 0 0 /dev/sdk 0 + # U46 ONLINE 0 0 0 /dev/sdm 0 + # U47 ONLINE 0 0 0 /dev/sdn 0 + # U50 ONLINE 0 0 0 /dev/sdbn 0 + # + ZPOOL_SCRIPTS_AS_ROOT=1 $ZPOOL status -c upath,fault_led "$pool" | grep '/dev/' | ( + rc=0 + while read -r vdev state _ _ _ therest; do + # Read out current LED value and path + # Get dev name (like 'sda') + dev=$(basename "$(echo "$therest" | awk '{print $(NF-1)}')") + vdev_enc_sysfs_path=$(realpath "/sys/class/block/$dev/device/enclosure_device"*) + if [ ! -d "$vdev_enc_sysfs_path" ] ; then + # This is not a JBOD disk, but it could be a PCI NVMe drive + vdev_enc_sysfs_path=$(nvme_dev_to_slot "$dev") + fi + + current_val=$(echo "$therest" | awk '{print $NF}') + + if [ "$current_val" != "0" ] ; then + current_val=1 + fi + + if [ -z "$vdev_enc_sysfs_path" ] ; then + # Skip anything with no sysfs LED entries + continue + fi + + led_path=$(path_to_led "$vdev_enc_sysfs_path") + if [ ! -e "$led_path" ] ; then + rc=3 + zed_log_msg "vdev $vdev '$led_path' doesn't exist" + continue + fi + + val=$(state_to_val "$state") + + if [ "$current_val" = "$val" ] ; then + # LED is already set correctly + continue + fi + + if ! check_and_set_led "$led_path" "$val"; then + rc=3 + fi + done + exit "$rc"; ) +} + +if [ -n "$ZEVENT_VDEV_ENC_SYSFS_PATH" ] && [ -n "$ZEVENT_VDEV_STATE_STR" ] ; then + # Got a statechange for an individual vdev + val=$(state_to_val "$ZEVENT_VDEV_STATE_STR") + vdev=$(basename "$ZEVENT_VDEV_PATH") + ledpath=$(path_to_led "$ZEVENT_VDEV_ENC_SYSFS_PATH") + check_and_set_led "$ledpath" "$val" +else + # Process the entire pool + poolname=$(zed_guid_to_pool "$ZEVENT_POOL_GUID") + process_pool "$poolname" +fi diff --git a/straper/db/public/zfs/etc-zfs/zed.d/resilver_finish-notify.sh b/straper/db/public/zfs/etc-zfs/zed.d/resilver_finish-notify.sh new file mode 100755 index 0000000..fc9dc23 --- /dev/null +++ b/straper/db/public/zfs/etc-zfs/zed.d/resilver_finish-notify.sh @@ -0,0 +1,60 @@ +#!/bin/sh +# shellcheck disable=SC2154 +# +# Send notification in response to a RESILVER_FINISH or SCRUB_FINISH. +# +# By default, "zpool status" output will only be included for a scrub_finish +# zevent if the pool is not healthy; to always include its output, set +# ZED_NOTIFY_VERBOSE=1. +# +# Exit codes: +# 0: notification sent +# 1: notification failed +# 2: notification not configured +# 3: notification suppressed +# 9: internal error + +[ -f "${ZED_ZEDLET_DIR}/zed.rc" ] && . "${ZED_ZEDLET_DIR}/zed.rc" +. "${ZED_ZEDLET_DIR}/zed-functions.sh" + +[ -n "${ZEVENT_POOL}" ] || exit 9 +[ -n "${ZEVENT_SUBCLASS}" ] || exit 9 + +if [ "${ZEVENT_SUBCLASS}" = "resilver_finish" ]; then + action="resilver" +elif [ "${ZEVENT_SUBCLASS}" = "scrub_finish" ]; then + action="scrub" +else + zed_log_err "unsupported event class \"${ZEVENT_SUBCLASS}\"" + exit 9 +fi + +zed_check_cmd "${ZPOOL}" || exit 9 + +# For scrub, suppress notification if the pool is healthy +# and verbosity is not enabled. +# +if [ "${ZEVENT_SUBCLASS}" = "scrub_finish" ]; then + healthy="$("${ZPOOL}" status -x "${ZEVENT_POOL}" \ + | grep "'${ZEVENT_POOL}' is healthy")" + [ -n "${healthy}" ] && [ "${ZED_NOTIFY_VERBOSE}" -eq 0 ] && exit 3 +fi + +umask 077 +note_subject="ZFS ${ZEVENT_SUBCLASS} event for ${ZEVENT_POOL} on $(hostname)" +note_pathname="$(mktemp)" +{ + echo "ZFS has finished a ${action}:" + echo + echo " eid: ${ZEVENT_EID}" + echo " class: ${ZEVENT_SUBCLASS}" + echo " host: $(hostname)" + echo " time: ${ZEVENT_TIME_STRING}" + + "${ZPOOL}" status "${ZEVENT_POOL}" + +} > "${note_pathname}" + +zed_notify "${note_subject}" "${note_pathname}"; rv=$? +rm -f "${note_pathname}" +exit "${rv}" diff --git a/straper/db/public/zfs/etc-zfs/zed.d/resilver_finish-start-scrub.sh b/straper/db/public/zfs/etc-zfs/zed.d/resilver_finish-start-scrub.sh new file mode 100755 index 0000000..cafce6f --- /dev/null +++ b/straper/db/public/zfs/etc-zfs/zed.d/resilver_finish-start-scrub.sh @@ -0,0 +1,20 @@ +#!/bin/sh +# shellcheck disable=SC2154 +# resilver_finish-start-scrub.sh +# Run a scrub after a resilver +# +# Exit codes: +# 1: Internal error +# 2: Script wasn't enabled in zed.rc +# 3: Scrubs are automatically started for sequential resilvers +[ -f "${ZED_ZEDLET_DIR}/zed.rc" ] && . "${ZED_ZEDLET_DIR}/zed.rc" +. "${ZED_ZEDLET_DIR}/zed-functions.sh" + +[ "${ZED_SCRUB_AFTER_RESILVER}" = "1" ] || exit 2 +[ "${ZEVENT_RESILVER_TYPE}" != "sequential" ] || exit 3 +[ -n "${ZEVENT_POOL}" ] || exit 1 +[ -n "${ZEVENT_SUBCLASS}" ] || exit 1 +zed_check_cmd "${ZPOOL}" || exit 1 + +zed_log_msg "Starting scrub after resilver on ${ZEVENT_POOL}" +"${ZPOOL}" scrub "${ZEVENT_POOL}" diff --git a/straper/db/public/zfs/etc-zfs/zed.d/scrub_finish-notify.sh b/straper/db/public/zfs/etc-zfs/zed.d/scrub_finish-notify.sh new file mode 100755 index 0000000..fc9dc23 --- /dev/null +++ b/straper/db/public/zfs/etc-zfs/zed.d/scrub_finish-notify.sh @@ -0,0 +1,60 @@ +#!/bin/sh +# shellcheck disable=SC2154 +# +# Send notification in response to a RESILVER_FINISH or SCRUB_FINISH. +# +# By default, "zpool status" output will only be included for a scrub_finish +# zevent if the pool is not healthy; to always include its output, set +# ZED_NOTIFY_VERBOSE=1. +# +# Exit codes: +# 0: notification sent +# 1: notification failed +# 2: notification not configured +# 3: notification suppressed +# 9: internal error + +[ -f "${ZED_ZEDLET_DIR}/zed.rc" ] && . "${ZED_ZEDLET_DIR}/zed.rc" +. "${ZED_ZEDLET_DIR}/zed-functions.sh" + +[ -n "${ZEVENT_POOL}" ] || exit 9 +[ -n "${ZEVENT_SUBCLASS}" ] || exit 9 + +if [ "${ZEVENT_SUBCLASS}" = "resilver_finish" ]; then + action="resilver" +elif [ "${ZEVENT_SUBCLASS}" = "scrub_finish" ]; then + action="scrub" +else + zed_log_err "unsupported event class \"${ZEVENT_SUBCLASS}\"" + exit 9 +fi + +zed_check_cmd "${ZPOOL}" || exit 9 + +# For scrub, suppress notification if the pool is healthy +# and verbosity is not enabled. +# +if [ "${ZEVENT_SUBCLASS}" = "scrub_finish" ]; then + healthy="$("${ZPOOL}" status -x "${ZEVENT_POOL}" \ + | grep "'${ZEVENT_POOL}' is healthy")" + [ -n "${healthy}" ] && [ "${ZED_NOTIFY_VERBOSE}" -eq 0 ] && exit 3 +fi + +umask 077 +note_subject="ZFS ${ZEVENT_SUBCLASS} event for ${ZEVENT_POOL} on $(hostname)" +note_pathname="$(mktemp)" +{ + echo "ZFS has finished a ${action}:" + echo + echo " eid: ${ZEVENT_EID}" + echo " class: ${ZEVENT_SUBCLASS}" + echo " host: $(hostname)" + echo " time: ${ZEVENT_TIME_STRING}" + + "${ZPOOL}" status "${ZEVENT_POOL}" + +} > "${note_pathname}" + +zed_notify "${note_subject}" "${note_pathname}"; rv=$? +rm -f "${note_pathname}" +exit "${rv}" diff --git a/straper/db/public/zfs/etc-zfs/zed.d/statechange-led.sh b/straper/db/public/zfs/etc-zfs/zed.d/statechange-led.sh new file mode 100755 index 0000000..40cb61f --- /dev/null +++ b/straper/db/public/zfs/etc-zfs/zed.d/statechange-led.sh @@ -0,0 +1,242 @@ +#!/bin/sh +# shellcheck disable=SC2154 +# +# Turn off/on vdevs' enclosure fault LEDs when their pool's state changes. +# +# Turn a vdev's fault LED on if it becomes FAULTED, DEGRADED or UNAVAIL. +# Turn its LED off when it's back ONLINE again. +# +# This script run in two basic modes: +# +# 1. If $ZEVENT_VDEV_ENC_SYSFS_PATH and $ZEVENT_VDEV_STATE_STR are set, then +# only set the LED for that particular vdev. This is the case for statechange +# events and some vdev_* events. +# +# 2. If those vars are not set, then check the state of all vdevs in the pool +# and set the LEDs accordingly. This is the case for pool_import events. +# +# Note that this script requires that your enclosure be supported by the +# Linux SCSI Enclosure services (SES) driver. The script will do nothing +# if you have no enclosure, or if your enclosure isn't supported. +# +# Exit codes: +# 0: enclosure led successfully set +# 1: enclosure leds not available +# 2: enclosure leds administratively disabled +# 3: The led sysfs path passed from ZFS does not exist +# 4: $ZPOOL not set +# 5: awk is not installed + +[ -f "${ZED_ZEDLET_DIR}/zed.rc" ] && . "${ZED_ZEDLET_DIR}/zed.rc" +. "${ZED_ZEDLET_DIR}/zed-functions.sh" + +if [ ! -d /sys/class/enclosure ] && [ ! -d /sys/bus/pci/slots ] ; then + # No JBOD enclosure or NVMe slots + exit 1 +fi + +if [ "${ZED_USE_ENCLOSURE_LEDS}" != "1" ] ; then + exit 2 +fi + +zed_check_cmd "$ZPOOL" || exit 4 +zed_check_cmd awk || exit 5 + +# Global used in set_led debug print +vdev="" + +# check_and_set_led (file, val) +# +# Read an enclosure sysfs file, and write it if it's not already set to 'val' +# +# Arguments +# file: sysfs file to set (like /sys/class/enclosure/0:0:1:0/SLOT 10/fault) +# val: value to set it to +# +# Return +# 0 on success, 3 on missing sysfs path +# +check_and_set_led() +{ + file="$1" + val="$2" + + if [ -z "$val" ]; then + return 0 + fi + + if [ ! -e "$file" ] ; then + return 3 + fi + + # If another process is accessing the LED when we attempt to update it, + # the update will be lost so retry until the LED actually changes or we + # timeout. + for _ in 1 2 3 4 5; do + # We want to check the current state first, since writing to the + # 'fault' entry always causes a SES command, even if the + # current state is already what you want. + read -r current < "${file}" + + # On some enclosures if you write 1 to fault, and read it back, + # it will return 2. Treat all non-zero values as 1 for + # simplicity. + if [ "$current" != "0" ] ; then + current=1 + fi + + if [ "$current" != "$val" ] ; then + echo "$val" > "$file" + zed_log_msg "vdev $vdev set '$file' LED to $val" + else + break + fi + done +} + +# Fault LEDs for JBODs and NVMe drives are handled a little differently. +# +# On JBODs the fault LED is called 'fault' and on a path like this: +# +# /sys/class/enclosure/0:0:1:0/SLOT 10/fault +# +# On NVMe it's called 'attention' and on a path like this: +# +# /sys/bus/pci/slot/0/attention +# +# This function returns the full path to the fault LED file for a given +# enclosure/slot directory. +# +path_to_led() +{ + dir=$1 + if [ -f "$dir/fault" ] ; then + echo "$dir/fault" + elif [ -f "$dir/attention" ] ; then + echo "$dir/attention" + fi +} + +state_to_val() +{ + state="$1" + case "$state" in + FAULTED|DEGRADED|UNAVAIL|REMOVED) + echo 1 + ;; + ONLINE) + echo 0 + ;; + *) + echo "invalid state: $state" + ;; + esac +} + +# +# Given a nvme name like 'nvme0n1', pass back its slot directory +# like "/sys/bus/pci/slots/0" +# +nvme_dev_to_slot() +{ + dev="$1" + + # Get the address "0000:01:00.0" + read -r address < "/sys/class/block/$dev/device/address" + + find /sys/bus/pci/slots -regex '.*/[0-9]+/address$' | \ + while read -r sys_addr; do + read -r this_address < "$sys_addr" + + # The format of address is a little different between + # /sys/class/block/$dev/device/address and + # /sys/bus/pci/slots/ + # + # address= "0000:01:00.0" + # this_address = "0000:01:00" + # + if echo "$address" | grep -Eq ^"$this_address" ; then + echo "${sys_addr%/*}" + break + fi + done +} + + +# process_pool (pool) +# +# Iterate through a pool and set the vdevs' enclosure slot LEDs to +# those vdevs' state. +# +# Arguments +# pool: Pool name. +# +# Return +# 0 on success, 3 on missing sysfs path +# +process_pool() +{ + pool="$1" + + # The output will be the vdevs only (from "grep '/dev/'"): + # + # U45 ONLINE 0 0 0 /dev/sdk 0 + # U46 ONLINE 0 0 0 /dev/sdm 0 + # U47 ONLINE 0 0 0 /dev/sdn 0 + # U50 ONLINE 0 0 0 /dev/sdbn 0 + # + ZPOOL_SCRIPTS_AS_ROOT=1 $ZPOOL status -c upath,fault_led "$pool" | grep '/dev/' | ( + rc=0 + while read -r vdev state _ _ _ therest; do + # Read out current LED value and path + # Get dev name (like 'sda') + dev=$(basename "$(echo "$therest" | awk '{print $(NF-1)}')") + vdev_enc_sysfs_path=$(realpath "/sys/class/block/$dev/device/enclosure_device"*) + if [ ! -d "$vdev_enc_sysfs_path" ] ; then + # This is not a JBOD disk, but it could be a PCI NVMe drive + vdev_enc_sysfs_path=$(nvme_dev_to_slot "$dev") + fi + + current_val=$(echo "$therest" | awk '{print $NF}') + + if [ "$current_val" != "0" ] ; then + current_val=1 + fi + + if [ -z "$vdev_enc_sysfs_path" ] ; then + # Skip anything with no sysfs LED entries + continue + fi + + led_path=$(path_to_led "$vdev_enc_sysfs_path") + if [ ! -e "$led_path" ] ; then + rc=3 + zed_log_msg "vdev $vdev '$led_path' doesn't exist" + continue + fi + + val=$(state_to_val "$state") + + if [ "$current_val" = "$val" ] ; then + # LED is already set correctly + continue + fi + + if ! check_and_set_led "$led_path" "$val"; then + rc=3 + fi + done + exit "$rc"; ) +} + +if [ -n "$ZEVENT_VDEV_ENC_SYSFS_PATH" ] && [ -n "$ZEVENT_VDEV_STATE_STR" ] ; then + # Got a statechange for an individual vdev + val=$(state_to_val "$ZEVENT_VDEV_STATE_STR") + vdev=$(basename "$ZEVENT_VDEV_PATH") + ledpath=$(path_to_led "$ZEVENT_VDEV_ENC_SYSFS_PATH") + check_and_set_led "$ledpath" "$val" +else + # Process the entire pool + poolname=$(zed_guid_to_pool "$ZEVENT_POOL_GUID") + process_pool "$poolname" +fi diff --git a/straper/db/public/zfs/etc-zfs/zed.d/statechange-notify.sh b/straper/db/public/zfs/etc-zfs/zed.d/statechange-notify.sh new file mode 100755 index 0000000..c8c8323 --- /dev/null +++ b/straper/db/public/zfs/etc-zfs/zed.d/statechange-notify.sh @@ -0,0 +1,77 @@ +#!/bin/sh +# SPDX-License-Identifier: CDDL-1.0 +# shellcheck disable=SC2154 +# +# CDDL HEADER START +# +# The contents of this file are subject to the terms of the +# Common Development and Distribution License Version 1.0 (CDDL-1.0). +# You can obtain a copy of the license from the top-level file +# "OPENSOLARIS.LICENSE" or at <http://opensource.org/licenses/CDDL-1.0>. +# You may not use this file except in compliance with the license. +# +# CDDL HEADER END +# + +# +# Send notification in response to a fault induced statechange +# +# ZEVENT_SUBCLASS: 'statechange' +# ZEVENT_VDEV_STATE_STR: 'DEGRADED', 'FAULTED', 'REMOVED', or 'UNAVAIL' +# +# Exit codes: +# 0: notification sent +# 1: notification failed +# 2: notification not configured +# 3: statechange not relevant +# 4: statechange string missing (unexpected) + +[ -f "${ZED_ZEDLET_DIR}/zed.rc" ] && . "${ZED_ZEDLET_DIR}/zed.rc" +. "${ZED_ZEDLET_DIR}/zed-functions.sh" + +[ -n "${ZEVENT_VDEV_STATE_STR}" ] || exit 4 + +if [ "${ZEVENT_VDEV_STATE_STR}" != "FAULTED" ] \ + && [ "${ZEVENT_VDEV_STATE_STR}" != "DEGRADED" ] \ + && [ "${ZEVENT_VDEV_STATE_STR}" != "REMOVED" ] \ + && [ "${ZEVENT_VDEV_STATE_STR}" != "UNAVAIL" ]; then + exit 3 +fi + +umask 077 +note_subject="ZFS device fault for pool ${ZEVENT_POOL} on $(hostname)" +note_pathname="$(mktemp)" +{ + if [ "${ZEVENT_VDEV_STATE_STR}" = "FAULTED" ] ; then + echo "The number of I/O errors associated with a ZFS device exceeded" + echo "acceptable levels. ZFS has marked the device as faulted." + elif [ "${ZEVENT_VDEV_STATE_STR}" = "DEGRADED" ] ; then + echo "The number of checksum errors associated with a ZFS device" + echo "exceeded acceptable levels. ZFS has marked the device as" + echo "degraded." + else + echo "ZFS has detected that a device was removed." + fi + + echo + echo " impact: Fault tolerance of the pool may be compromised." + echo " eid: ${ZEVENT_EID}" + echo " class: ${ZEVENT_SUBCLASS}" + echo " state: ${ZEVENT_VDEV_STATE_STR}" + echo " host: $(hostname)" + echo " time: ${ZEVENT_TIME_STRING}" + + [ -n "${ZEVENT_VDEV_TYPE}" ] && echo " vtype: ${ZEVENT_VDEV_TYPE}" + [ -n "${ZEVENT_VDEV_PATH}" ] && echo " vpath: ${ZEVENT_VDEV_PATH}" + [ -n "${ZEVENT_VDEV_PHYSPATH}" ] && echo " vphys: ${ZEVENT_VDEV_PHYSPATH}" + [ -n "${ZEVENT_VDEV_GUID}" ] && echo " vguid: ${ZEVENT_VDEV_GUID}" + [ -n "${ZEVENT_VDEV_DEVID}" ] && echo " devid: ${ZEVENT_VDEV_DEVID}" + + echo " pool: ${ZEVENT_POOL} (${ZEVENT_POOL_GUID})" + +} > "${note_pathname}" + +zed_notify "${note_subject}" "${note_pathname}"; rv=$? + +rm -f "${note_pathname}" +exit "${rv}" diff --git a/straper/db/public/zfs/etc-zfs/zed.d/statechange-slot_off.sh b/straper/db/public/zfs/etc-zfs/zed.d/statechange-slot_off.sh new file mode 100755 index 0000000..06acce9 --- /dev/null +++ b/straper/db/public/zfs/etc-zfs/zed.d/statechange-slot_off.sh @@ -0,0 +1,64 @@ +#!/bin/sh +# shellcheck disable=SC3014,SC2154,SC2086,SC2034 +# +# Turn off disk's enclosure slot if it becomes FAULTED. +# +# Bad SCSI disks can often "disappear and reappear" causing all sorts of chaos +# as they flip between FAULTED and ONLINE. If +# ZED_POWER_OFF_ENCLOSURE_SLOT_ON_FAULT is set in zed.rc, and the disk gets +# FAULTED, then power down the slot via sysfs: +# +# /sys/class/enclosure/<enclosure>/<slot>/power_status +# +# We assume the user will be responsible for turning the slot back on again. +# +# Note that this script requires that your enclosure be supported by the +# Linux SCSI Enclosure services (SES) driver. The script will do nothing +# if you have no enclosure, or if your enclosure isn't supported. +# +# Exit codes: +# 0: slot successfully powered off +# 1: enclosure not available +# 2: ZED_POWER_OFF_ENCLOSURE_SLOT_ON_FAULT disabled +# 3: vdev was not FAULTED +# 4: The enclosure sysfs path passed from ZFS does not exist +# 5: Enclosure slot didn't actually turn off after we told it to + +[ -f "${ZED_ZEDLET_DIR}/zed.rc" ] && . "${ZED_ZEDLET_DIR}/zed.rc" +. "${ZED_ZEDLET_DIR}/zed-functions.sh" + +if [ ! -d /sys/class/enclosure ] ; then + # No JBOD enclosure or NVMe slots + exit 1 +fi + +if [ "${ZED_POWER_OFF_ENCLOSURE_SLOT_ON_FAULT}" != "1" ] ; then + exit 2 +fi + +if [ "$ZEVENT_VDEV_STATE_STR" != "FAULTED" ] ; then + exit 3 +fi + +if [ ! -f "$ZEVENT_VDEV_ENC_SYSFS_PATH/power_status" ] ; then + exit 4 +fi + +# Turn off the slot and wait for sysfs to report that the slot is off. +# It can take ~400ms on some enclosures and multiple retries may be needed. +for i in $(seq 1 20) ; do + echo "off" | tee "$ZEVENT_VDEV_ENC_SYSFS_PATH/power_status" + + for j in $(seq 1 5) ; do + if [ "$(cat $ZEVENT_VDEV_ENC_SYSFS_PATH/power_status)" == "off" ] ; then + break 2 + fi + sleep 0.1 + done +done + +if [ "$(cat $ZEVENT_VDEV_ENC_SYSFS_PATH/power_status)" != "off" ] ; then + exit 5 +fi + +zed_log_msg "powered down slot $ZEVENT_VDEV_ENC_SYSFS_PATH for $ZEVENT_VDEV_PATH" diff --git a/straper/db/public/zfs/etc-zfs/zed.d/vdev_attach-led.sh b/straper/db/public/zfs/etc-zfs/zed.d/vdev_attach-led.sh new file mode 100755 index 0000000..40cb61f --- /dev/null +++ b/straper/db/public/zfs/etc-zfs/zed.d/vdev_attach-led.sh @@ -0,0 +1,242 @@ +#!/bin/sh +# shellcheck disable=SC2154 +# +# Turn off/on vdevs' enclosure fault LEDs when their pool's state changes. +# +# Turn a vdev's fault LED on if it becomes FAULTED, DEGRADED or UNAVAIL. +# Turn its LED off when it's back ONLINE again. +# +# This script run in two basic modes: +# +# 1. If $ZEVENT_VDEV_ENC_SYSFS_PATH and $ZEVENT_VDEV_STATE_STR are set, then +# only set the LED for that particular vdev. This is the case for statechange +# events and some vdev_* events. +# +# 2. If those vars are not set, then check the state of all vdevs in the pool +# and set the LEDs accordingly. This is the case for pool_import events. +# +# Note that this script requires that your enclosure be supported by the +# Linux SCSI Enclosure services (SES) driver. The script will do nothing +# if you have no enclosure, or if your enclosure isn't supported. +# +# Exit codes: +# 0: enclosure led successfully set +# 1: enclosure leds not available +# 2: enclosure leds administratively disabled +# 3: The led sysfs path passed from ZFS does not exist +# 4: $ZPOOL not set +# 5: awk is not installed + +[ -f "${ZED_ZEDLET_DIR}/zed.rc" ] && . "${ZED_ZEDLET_DIR}/zed.rc" +. "${ZED_ZEDLET_DIR}/zed-functions.sh" + +if [ ! -d /sys/class/enclosure ] && [ ! -d /sys/bus/pci/slots ] ; then + # No JBOD enclosure or NVMe slots + exit 1 +fi + +if [ "${ZED_USE_ENCLOSURE_LEDS}" != "1" ] ; then + exit 2 +fi + +zed_check_cmd "$ZPOOL" || exit 4 +zed_check_cmd awk || exit 5 + +# Global used in set_led debug print +vdev="" + +# check_and_set_led (file, val) +# +# Read an enclosure sysfs file, and write it if it's not already set to 'val' +# +# Arguments +# file: sysfs file to set (like /sys/class/enclosure/0:0:1:0/SLOT 10/fault) +# val: value to set it to +# +# Return +# 0 on success, 3 on missing sysfs path +# +check_and_set_led() +{ + file="$1" + val="$2" + + if [ -z "$val" ]; then + return 0 + fi + + if [ ! -e "$file" ] ; then + return 3 + fi + + # If another process is accessing the LED when we attempt to update it, + # the update will be lost so retry until the LED actually changes or we + # timeout. + for _ in 1 2 3 4 5; do + # We want to check the current state first, since writing to the + # 'fault' entry always causes a SES command, even if the + # current state is already what you want. + read -r current < "${file}" + + # On some enclosures if you write 1 to fault, and read it back, + # it will return 2. Treat all non-zero values as 1 for + # simplicity. + if [ "$current" != "0" ] ; then + current=1 + fi + + if [ "$current" != "$val" ] ; then + echo "$val" > "$file" + zed_log_msg "vdev $vdev set '$file' LED to $val" + else + break + fi + done +} + +# Fault LEDs for JBODs and NVMe drives are handled a little differently. +# +# On JBODs the fault LED is called 'fault' and on a path like this: +# +# /sys/class/enclosure/0:0:1:0/SLOT 10/fault +# +# On NVMe it's called 'attention' and on a path like this: +# +# /sys/bus/pci/slot/0/attention +# +# This function returns the full path to the fault LED file for a given +# enclosure/slot directory. +# +path_to_led() +{ + dir=$1 + if [ -f "$dir/fault" ] ; then + echo "$dir/fault" + elif [ -f "$dir/attention" ] ; then + echo "$dir/attention" + fi +} + +state_to_val() +{ + state="$1" + case "$state" in + FAULTED|DEGRADED|UNAVAIL|REMOVED) + echo 1 + ;; + ONLINE) + echo 0 + ;; + *) + echo "invalid state: $state" + ;; + esac +} + +# +# Given a nvme name like 'nvme0n1', pass back its slot directory +# like "/sys/bus/pci/slots/0" +# +nvme_dev_to_slot() +{ + dev="$1" + + # Get the address "0000:01:00.0" + read -r address < "/sys/class/block/$dev/device/address" + + find /sys/bus/pci/slots -regex '.*/[0-9]+/address$' | \ + while read -r sys_addr; do + read -r this_address < "$sys_addr" + + # The format of address is a little different between + # /sys/class/block/$dev/device/address and + # /sys/bus/pci/slots/ + # + # address= "0000:01:00.0" + # this_address = "0000:01:00" + # + if echo "$address" | grep -Eq ^"$this_address" ; then + echo "${sys_addr%/*}" + break + fi + done +} + + +# process_pool (pool) +# +# Iterate through a pool and set the vdevs' enclosure slot LEDs to +# those vdevs' state. +# +# Arguments +# pool: Pool name. +# +# Return +# 0 on success, 3 on missing sysfs path +# +process_pool() +{ + pool="$1" + + # The output will be the vdevs only (from "grep '/dev/'"): + # + # U45 ONLINE 0 0 0 /dev/sdk 0 + # U46 ONLINE 0 0 0 /dev/sdm 0 + # U47 ONLINE 0 0 0 /dev/sdn 0 + # U50 ONLINE 0 0 0 /dev/sdbn 0 + # + ZPOOL_SCRIPTS_AS_ROOT=1 $ZPOOL status -c upath,fault_led "$pool" | grep '/dev/' | ( + rc=0 + while read -r vdev state _ _ _ therest; do + # Read out current LED value and path + # Get dev name (like 'sda') + dev=$(basename "$(echo "$therest" | awk '{print $(NF-1)}')") + vdev_enc_sysfs_path=$(realpath "/sys/class/block/$dev/device/enclosure_device"*) + if [ ! -d "$vdev_enc_sysfs_path" ] ; then + # This is not a JBOD disk, but it could be a PCI NVMe drive + vdev_enc_sysfs_path=$(nvme_dev_to_slot "$dev") + fi + + current_val=$(echo "$therest" | awk '{print $NF}') + + if [ "$current_val" != "0" ] ; then + current_val=1 + fi + + if [ -z "$vdev_enc_sysfs_path" ] ; then + # Skip anything with no sysfs LED entries + continue + fi + + led_path=$(path_to_led "$vdev_enc_sysfs_path") + if [ ! -e "$led_path" ] ; then + rc=3 + zed_log_msg "vdev $vdev '$led_path' doesn't exist" + continue + fi + + val=$(state_to_val "$state") + + if [ "$current_val" = "$val" ] ; then + # LED is already set correctly + continue + fi + + if ! check_and_set_led "$led_path" "$val"; then + rc=3 + fi + done + exit "$rc"; ) +} + +if [ -n "$ZEVENT_VDEV_ENC_SYSFS_PATH" ] && [ -n "$ZEVENT_VDEV_STATE_STR" ] ; then + # Got a statechange for an individual vdev + val=$(state_to_val "$ZEVENT_VDEV_STATE_STR") + vdev=$(basename "$ZEVENT_VDEV_PATH") + ledpath=$(path_to_led "$ZEVENT_VDEV_ENC_SYSFS_PATH") + check_and_set_led "$ledpath" "$val" +else + # Process the entire pool + poolname=$(zed_guid_to_pool "$ZEVENT_POOL_GUID") + process_pool "$poolname" +fi diff --git a/straper/db/public/zfs/etc-zfs/zed.d/vdev_clear-led.sh b/straper/db/public/zfs/etc-zfs/zed.d/vdev_clear-led.sh new file mode 100755 index 0000000..40cb61f --- /dev/null +++ b/straper/db/public/zfs/etc-zfs/zed.d/vdev_clear-led.sh @@ -0,0 +1,242 @@ +#!/bin/sh +# shellcheck disable=SC2154 +# +# Turn off/on vdevs' enclosure fault LEDs when their pool's state changes. +# +# Turn a vdev's fault LED on if it becomes FAULTED, DEGRADED or UNAVAIL. +# Turn its LED off when it's back ONLINE again. +# +# This script run in two basic modes: +# +# 1. If $ZEVENT_VDEV_ENC_SYSFS_PATH and $ZEVENT_VDEV_STATE_STR are set, then +# only set the LED for that particular vdev. This is the case for statechange +# events and some vdev_* events. +# +# 2. If those vars are not set, then check the state of all vdevs in the pool +# and set the LEDs accordingly. This is the case for pool_import events. +# +# Note that this script requires that your enclosure be supported by the +# Linux SCSI Enclosure services (SES) driver. The script will do nothing +# if you have no enclosure, or if your enclosure isn't supported. +# +# Exit codes: +# 0: enclosure led successfully set +# 1: enclosure leds not available +# 2: enclosure leds administratively disabled +# 3: The led sysfs path passed from ZFS does not exist +# 4: $ZPOOL not set +# 5: awk is not installed + +[ -f "${ZED_ZEDLET_DIR}/zed.rc" ] && . "${ZED_ZEDLET_DIR}/zed.rc" +. "${ZED_ZEDLET_DIR}/zed-functions.sh" + +if [ ! -d /sys/class/enclosure ] && [ ! -d /sys/bus/pci/slots ] ; then + # No JBOD enclosure or NVMe slots + exit 1 +fi + +if [ "${ZED_USE_ENCLOSURE_LEDS}" != "1" ] ; then + exit 2 +fi + +zed_check_cmd "$ZPOOL" || exit 4 +zed_check_cmd awk || exit 5 + +# Global used in set_led debug print +vdev="" + +# check_and_set_led (file, val) +# +# Read an enclosure sysfs file, and write it if it's not already set to 'val' +# +# Arguments +# file: sysfs file to set (like /sys/class/enclosure/0:0:1:0/SLOT 10/fault) +# val: value to set it to +# +# Return +# 0 on success, 3 on missing sysfs path +# +check_and_set_led() +{ + file="$1" + val="$2" + + if [ -z "$val" ]; then + return 0 + fi + + if [ ! -e "$file" ] ; then + return 3 + fi + + # If another process is accessing the LED when we attempt to update it, + # the update will be lost so retry until the LED actually changes or we + # timeout. + for _ in 1 2 3 4 5; do + # We want to check the current state first, since writing to the + # 'fault' entry always causes a SES command, even if the + # current state is already what you want. + read -r current < "${file}" + + # On some enclosures if you write 1 to fault, and read it back, + # it will return 2. Treat all non-zero values as 1 for + # simplicity. + if [ "$current" != "0" ] ; then + current=1 + fi + + if [ "$current" != "$val" ] ; then + echo "$val" > "$file" + zed_log_msg "vdev $vdev set '$file' LED to $val" + else + break + fi + done +} + +# Fault LEDs for JBODs and NVMe drives are handled a little differently. +# +# On JBODs the fault LED is called 'fault' and on a path like this: +# +# /sys/class/enclosure/0:0:1:0/SLOT 10/fault +# +# On NVMe it's called 'attention' and on a path like this: +# +# /sys/bus/pci/slot/0/attention +# +# This function returns the full path to the fault LED file for a given +# enclosure/slot directory. +# +path_to_led() +{ + dir=$1 + if [ -f "$dir/fault" ] ; then + echo "$dir/fault" + elif [ -f "$dir/attention" ] ; then + echo "$dir/attention" + fi +} + +state_to_val() +{ + state="$1" + case "$state" in + FAULTED|DEGRADED|UNAVAIL|REMOVED) + echo 1 + ;; + ONLINE) + echo 0 + ;; + *) + echo "invalid state: $state" + ;; + esac +} + +# +# Given a nvme name like 'nvme0n1', pass back its slot directory +# like "/sys/bus/pci/slots/0" +# +nvme_dev_to_slot() +{ + dev="$1" + + # Get the address "0000:01:00.0" + read -r address < "/sys/class/block/$dev/device/address" + + find /sys/bus/pci/slots -regex '.*/[0-9]+/address$' | \ + while read -r sys_addr; do + read -r this_address < "$sys_addr" + + # The format of address is a little different between + # /sys/class/block/$dev/device/address and + # /sys/bus/pci/slots/ + # + # address= "0000:01:00.0" + # this_address = "0000:01:00" + # + if echo "$address" | grep -Eq ^"$this_address" ; then + echo "${sys_addr%/*}" + break + fi + done +} + + +# process_pool (pool) +# +# Iterate through a pool and set the vdevs' enclosure slot LEDs to +# those vdevs' state. +# +# Arguments +# pool: Pool name. +# +# Return +# 0 on success, 3 on missing sysfs path +# +process_pool() +{ + pool="$1" + + # The output will be the vdevs only (from "grep '/dev/'"): + # + # U45 ONLINE 0 0 0 /dev/sdk 0 + # U46 ONLINE 0 0 0 /dev/sdm 0 + # U47 ONLINE 0 0 0 /dev/sdn 0 + # U50 ONLINE 0 0 0 /dev/sdbn 0 + # + ZPOOL_SCRIPTS_AS_ROOT=1 $ZPOOL status -c upath,fault_led "$pool" | grep '/dev/' | ( + rc=0 + while read -r vdev state _ _ _ therest; do + # Read out current LED value and path + # Get dev name (like 'sda') + dev=$(basename "$(echo "$therest" | awk '{print $(NF-1)}')") + vdev_enc_sysfs_path=$(realpath "/sys/class/block/$dev/device/enclosure_device"*) + if [ ! -d "$vdev_enc_sysfs_path" ] ; then + # This is not a JBOD disk, but it could be a PCI NVMe drive + vdev_enc_sysfs_path=$(nvme_dev_to_slot "$dev") + fi + + current_val=$(echo "$therest" | awk '{print $NF}') + + if [ "$current_val" != "0" ] ; then + current_val=1 + fi + + if [ -z "$vdev_enc_sysfs_path" ] ; then + # Skip anything with no sysfs LED entries + continue + fi + + led_path=$(path_to_led "$vdev_enc_sysfs_path") + if [ ! -e "$led_path" ] ; then + rc=3 + zed_log_msg "vdev $vdev '$led_path' doesn't exist" + continue + fi + + val=$(state_to_val "$state") + + if [ "$current_val" = "$val" ] ; then + # LED is already set correctly + continue + fi + + if ! check_and_set_led "$led_path" "$val"; then + rc=3 + fi + done + exit "$rc"; ) +} + +if [ -n "$ZEVENT_VDEV_ENC_SYSFS_PATH" ] && [ -n "$ZEVENT_VDEV_STATE_STR" ] ; then + # Got a statechange for an individual vdev + val=$(state_to_val "$ZEVENT_VDEV_STATE_STR") + vdev=$(basename "$ZEVENT_VDEV_PATH") + ledpath=$(path_to_led "$ZEVENT_VDEV_ENC_SYSFS_PATH") + check_and_set_led "$ledpath" "$val" +else + # Process the entire pool + poolname=$(zed_guid_to_pool "$ZEVENT_POOL_GUID") + process_pool "$poolname" +fi diff --git a/straper/db/public/zfs/etc-zfs/zed.d/zed-functions.sh b/straper/db/public/zfs/etc-zfs/zed.d/zed-functions.sh new file mode 100644 index 0000000..6e00f15 --- /dev/null +++ b/straper/db/public/zfs/etc-zfs/zed.d/zed-functions.sh @@ -0,0 +1,824 @@ +#!/bin/sh +# shellcheck disable=SC2154,SC3043 +# zed-functions.sh +# +# ZED helper functions for use in ZEDLETs + + +# Variable Defaults +# +: "${ZED_LOCKDIR:="/var/lock"}" +: "${ZED_NOTIFY_INTERVAL_SECS:=3600}" +: "${ZED_NOTIFY_VERBOSE:=0}" +: "${ZED_RUNDIR:="/var/run"}" +: "${ZED_SYSLOG_PRIORITY:="daemon.notice"}" +: "${ZED_SYSLOG_TAG:="zed"}" + +ZED_FLOCK_FD=8 + + +# zed_check_cmd (cmd, ...) +# +# For each argument given, search PATH for the executable command [cmd]. +# Log a message if [cmd] is not found. +# +# Arguments +# cmd: name of executable command for which to search +# +# Return +# 0 if all commands are found in PATH and are executable +# n for a count of the command executables that are not found +# +zed_check_cmd() +{ + local cmd + local rv=0 + + for cmd; do + if ! command -v "${cmd}" >/dev/null 2>&1; then + zed_log_err "\"${cmd}\" not installed" + rv=$((rv + 1)) + fi + done + return "${rv}" +} + + +# zed_log_msg (msg, ...) +# +# Write all argument strings to the system log. +# +# Globals +# ZED_SYSLOG_PRIORITY +# ZED_SYSLOG_TAG +# +# Return +# nothing +# +zed_log_msg() +{ + logger -p "${ZED_SYSLOG_PRIORITY}" -t "${ZED_SYSLOG_TAG}" -- "$@" +} + + +# zed_log_err (msg, ...) +# +# Write an error message to the system log. This message will contain the +# script name, EID, and all argument strings. +# +# Globals +# ZED_SYSLOG_PRIORITY +# ZED_SYSLOG_TAG +# ZEVENT_EID +# +# Return +# nothing +# +zed_log_err() +{ + zed_log_msg "error: ${0##*/}:""${ZEVENT_EID:+" eid=${ZEVENT_EID}:"}" "$@" +} + + +# zed_lock (lockfile, [fd]) +# +# Obtain an exclusive (write) lock on [lockfile]. If the lock cannot be +# immediately acquired, wait until it becomes available. +# +# Every zed_lock() must be paired with a corresponding zed_unlock(). +# +# By default, flock-style locks associate the lockfile with file descriptor 8. +# The bash manpage warns that file descriptors >9 should be used with care as +# they may conflict with file descriptors used internally by the shell. File +# descriptor 9 is reserved for zed_rate_limit(). If concurrent locks are held +# within the same process, they must use different file descriptors (preferably +# decrementing from 8); otherwise, obtaining a new lock with a given file +# descriptor will release the previous lock associated with that descriptor. +# +# Arguments +# lockfile: pathname of the lock file; the lock will be stored in +# ZED_LOCKDIR unless the pathname contains a "/". +# fd: integer for the file descriptor used by flock (OPTIONAL unless holding +# concurrent locks) +# +# Globals +# ZED_FLOCK_FD +# ZED_LOCKDIR +# +# Return +# nothing +# +zed_lock() +{ + local lockfile="$1" + local fd="${2:-${ZED_FLOCK_FD}}" + local umask_bak + local err + + [ -n "${lockfile}" ] || return + if ! expr "${lockfile}" : '.*/' >/dev/null 2>&1; then + lockfile="${ZED_LOCKDIR}/${lockfile}" + fi + + umask_bak="$(umask)" + umask 077 + + # Obtain a lock on the file bound to the given file descriptor. + # + eval "exec ${fd}>> '${lockfile}'" + if ! err="$(flock --exclusive "${fd}" 2>&1)"; then + zed_log_err "failed to lock \"${lockfile}\": ${err}" + fi + + umask "${umask_bak}" +} + + +# zed_unlock (lockfile, [fd]) +# +# Release the lock on [lockfile]. +# +# Arguments +# lockfile: pathname of the lock file +# fd: integer for the file descriptor used by flock (must match the file +# descriptor passed to the zed_lock function call) +# +# Globals +# ZED_FLOCK_FD +# ZED_LOCKDIR +# +# Return +# nothing +# +zed_unlock() +{ + local lockfile="$1" + local fd="${2:-${ZED_FLOCK_FD}}" + local err + + [ -n "${lockfile}" ] || return + if ! expr "${lockfile}" : '.*/' >/dev/null 2>&1; then + lockfile="${ZED_LOCKDIR}/${lockfile}" + fi + + # Release the lock and close the file descriptor. + if ! err="$(flock --unlock "${fd}" 2>&1)"; then + zed_log_err "failed to unlock \"${lockfile}\": ${err}" + fi + eval "exec ${fd}>&-" +} + + +# zed_notify (subject, pathname) +# +# Send a notification via all available methods. +# +# Arguments +# subject: notification subject +# pathname: pathname containing the notification message (OPTIONAL) +# +# Return +# 0: notification succeeded via at least one method +# 1: notification failed +# 2: no notification methods configured +# +zed_notify() +{ + local subject="$1" + local pathname="$2" + local num_success=0 + local num_failure=0 + + zed_notify_email "${subject}" "${pathname}"; rv=$? + [ "${rv}" -eq 0 ] && num_success=$((num_success + 1)) + [ "${rv}" -eq 1 ] && num_failure=$((num_failure + 1)) + + zed_notify_pushbullet "${subject}" "${pathname}"; rv=$? + [ "${rv}" -eq 0 ] && num_success=$((num_success + 1)) + [ "${rv}" -eq 1 ] && num_failure=$((num_failure + 1)) + + zed_notify_slack_webhook "${subject}" "${pathname}"; rv=$? + [ "${rv}" -eq 0 ] && num_success=$((num_success + 1)) + [ "${rv}" -eq 1 ] && num_failure=$((num_failure + 1)) + + zed_notify_pushover "${subject}" "${pathname}"; rv=$? + [ "${rv}" -eq 0 ] && num_success=$((num_success + 1)) + [ "${rv}" -eq 1 ] && num_failure=$((num_failure + 1)) + + zed_notify_ntfy "${subject}" "${pathname}"; rv=$? + [ "${rv}" -eq 0 ] && num_success=$((num_success + 1)) + [ "${rv}" -eq 1 ] && num_failure=$((num_failure + 1)) + + zed_notify_gotify "${subject}" "${pathname}"; rv=$? + [ "${rv}" -eq 0 ] && num_success=$((num_success + 1)) + [ "${rv}" -eq 1 ] && num_failure=$((num_failure + 1)) + + [ "${num_success}" -gt 0 ] && return 0 + [ "${num_failure}" -gt 0 ] && return 1 + return 2 +} + + +# zed_notify_email (subject, pathname) +# +# Send a notification via email to the address specified by ZED_EMAIL_ADDR. +# +# Requires the mail executable to be installed in the standard PATH, or +# ZED_EMAIL_PROG to be defined with the pathname of an executable capable of +# reading a message body from stdin. +# +# Command-line options to the mail executable can be specified in +# ZED_EMAIL_OPTS. This undergoes the following keyword substitutions: +# - @ADDRESS@ is replaced with the space-delimited recipient email address(es) +# - @SUBJECT@ is replaced with the notification subject +# If @SUBJECT@ was omited here, a "Subject: ..." header will be added to notification +# +# +# Arguments +# subject: notification subject +# pathname: pathname containing the notification message (OPTIONAL) +# +# Globals +# ZED_EMAIL_PROG +# ZED_EMAIL_OPTS +# ZED_EMAIL_ADDR +# +# Return +# 0: notification sent +# 1: notification failed +# 2: not configured +# +zed_notify_email() +{ + local subject="${1:-"ZED notification"}" + local pathname="${2:-"/dev/null"}" + + : "${ZED_EMAIL_PROG:="mail"}" + : "${ZED_EMAIL_OPTS:="-s '@SUBJECT@' @ADDRESS@"}" + + # For backward compatibility with ZED_EMAIL. + if [ -n "${ZED_EMAIL}" ] && [ -z "${ZED_EMAIL_ADDR}" ]; then + ZED_EMAIL_ADDR="${ZED_EMAIL}" + fi + [ -n "${ZED_EMAIL_ADDR}" ] || return 2 + + zed_check_cmd "${ZED_EMAIL_PROG}" || return 1 + + [ -n "${subject}" ] || return 1 + if [ ! -r "${pathname}" ]; then + zed_log_err \ + "${ZED_EMAIL_PROG##*/} cannot read \"${pathname}\"" + return 1 + fi + + # construct cmdline options + ZED_EMAIL_OPTS_PARSED="$(echo "${ZED_EMAIL_OPTS}" \ + | sed -e "s/@ADDRESS@/${ZED_EMAIL_ADDR}/g" \ + -e "s/@SUBJECT@/${subject}/g")" + + # pipe message to email prog + # shellcheck disable=SC2086,SC2248 + { + # no subject passed as option? + if [ "${ZED_EMAIL_OPTS%@SUBJECT@*}" = "${ZED_EMAIL_OPTS}" ] ; then + # inject subject header + printf "Subject: %s\n" "${subject}" + # The following empty line is needed to separate the header from the + # body of the message. Otherwise programs like sendmail will skip + # everything up to the first empty line (or wont send an email at + # all) and will still exit with exit code 0 + printf "\n" + fi + # output message + cat "${pathname}" + } | + eval ${ZED_EMAIL_PROG} ${ZED_EMAIL_OPTS_PARSED} >/dev/null 2>&1 + rv=$? + if [ "${rv}" -ne 0 ]; then + zed_log_err "${ZED_EMAIL_PROG##*/} exit=${rv}" + return 1 + fi + return 0 +} + + +# zed_notify_pushbullet (subject, pathname) +# +# Send a notification via Pushbullet <https://www.pushbullet.com/>. +# The access token (ZED_PUSHBULLET_ACCESS_TOKEN) identifies this client to the +# Pushbullet server. The optional channel tag (ZED_PUSHBULLET_CHANNEL_TAG) is +# for pushing to notification feeds that can be subscribed to; if a channel is +# not defined, push notifications will instead be sent to all devices +# associated with the account specified by the access token. +# +# Requires awk, curl, and sed executables to be installed in the standard PATH. +# +# References +# https://docs.pushbullet.com/ +# https://www.pushbullet.com/security +# +# Arguments +# subject: notification subject +# pathname: pathname containing the notification message (OPTIONAL) +# +# Globals +# ZED_PUSHBULLET_ACCESS_TOKEN +# ZED_PUSHBULLET_CHANNEL_TAG +# +# Return +# 0: notification sent +# 1: notification failed +# 2: not configured +# +zed_notify_pushbullet() +{ + local subject="$1" + local pathname="${2:-"/dev/null"}" + local msg_body + local msg_tag + local msg_json + local msg_out + local msg_err + local url="https://api.pushbullet.com/v2/pushes" + + [ -n "${ZED_PUSHBULLET_ACCESS_TOKEN}" ] || return 2 + + [ -n "${subject}" ] || return 1 + if [ ! -r "${pathname}" ]; then + zed_log_err "pushbullet cannot read \"${pathname}\"" + return 1 + fi + + zed_check_cmd "awk" "curl" "sed" || return 1 + + # Escape the following characters in the message body for JSON: + # newline, backslash, double quote, horizontal tab, vertical tab, + # and carriage return. + # + msg_body="$(awk '{ ORS="\\n" } { gsub(/\\/, "\\\\"); gsub(/"/, "\\\""); + gsub(/\t/, "\\t"); gsub(/\f/, "\\f"); gsub(/\r/, "\\r"); print }' \ + "${pathname}")" + + # Push to a channel if one is configured. + # + [ -n "${ZED_PUSHBULLET_CHANNEL_TAG}" ] && msg_tag="$(printf \ + '"channel_tag": "%s", ' "${ZED_PUSHBULLET_CHANNEL_TAG}")" + + # Construct the JSON message for pushing a note. + # + msg_json="$(printf '{%s"type": "note", "title": "%s", "body": "%s"}' \ + "${msg_tag}" "${subject}" "${msg_body}")" + + # Send the POST request and check for errors. + # + msg_out="$(curl -u "${ZED_PUSHBULLET_ACCESS_TOKEN}:" -X POST "${url}" \ + --header "Content-Type: application/json" --data-binary "${msg_json}" \ + 2>/dev/null)"; rv=$? + if [ "${rv}" -ne 0 ]; then + zed_log_err "curl exit=${rv}" + return 1 + fi + msg_err="$(echo "${msg_out}" \ + | sed -n -e 's/.*"error" *:.*"message" *: *"\([^"]*\)".*/\1/p')" + if [ -n "${msg_err}" ]; then + zed_log_err "pushbullet \"${msg_err}"\" + return 1 + fi + return 0 +} + + +# zed_notify_slack_webhook (subject, pathname) +# +# Notification via Slack Webhook <https://api.slack.com/incoming-webhooks>. +# The Webhook URL (ZED_SLACK_WEBHOOK_URL) identifies this client to the +# Slack channel. +# +# Requires awk, curl, and sed executables to be installed in the standard PATH. +# +# References +# https://api.slack.com/incoming-webhooks +# +# Arguments +# subject: notification subject +# pathname: pathname containing the notification message (OPTIONAL) +# +# Globals +# ZED_SLACK_WEBHOOK_URL +# +# Return +# 0: notification sent +# 1: notification failed +# 2: not configured +# +zed_notify_slack_webhook() +{ + [ -n "${ZED_SLACK_WEBHOOK_URL}" ] || return 2 + + local subject="$1" + local pathname="${2:-"/dev/null"}" + local msg_body + local msg_tag + local msg_json + local msg_out + local msg_err + local url="${ZED_SLACK_WEBHOOK_URL}" + + [ -n "${subject}" ] || return 1 + if [ ! -r "${pathname}" ]; then + zed_log_err "slack webhook cannot read \"${pathname}\"" + return 1 + fi + + zed_check_cmd "awk" "curl" "sed" || return 1 + + # Escape the following characters in the message body for JSON: + # newline, backslash, double quote, horizontal tab, vertical tab, + # and carriage return. + # + msg_body="$(awk '{ ORS="\\n" } { gsub(/\\/, "\\\\"); gsub(/"/, "\\\""); + gsub(/\t/, "\\t"); gsub(/\f/, "\\f"); gsub(/\r/, "\\r"); print }' \ + "${pathname}")" + + # Construct the JSON message for posting. + # + msg_json="$(printf '{"text": "*%s*\\n%s"}' "${subject}" "${msg_body}" )" + + # Send the POST request and check for errors. + # + msg_out="$(curl -X POST "${url}" \ + --header "Content-Type: application/json" --data-binary "${msg_json}" \ + 2>/dev/null)"; rv=$? + if [ "${rv}" -ne 0 ]; then + zed_log_err "curl exit=${rv}" + return 1 + fi + msg_err="$(echo "${msg_out}" \ + | sed -n -e 's/.*"error" *:.*"message" *: *"\([^"]*\)".*/\1/p')" + if [ -n "${msg_err}" ]; then + zed_log_err "slack webhook \"${msg_err}"\" + return 1 + fi + return 0 +} + +# zed_notify_pushover (subject, pathname) +# +# Send a notification via Pushover <https://pushover.net/>. +# The access token (ZED_PUSHOVER_TOKEN) identifies this client to the +# Pushover server. The user token (ZED_PUSHOVER_USER) defines the user or +# group to which the notification will be sent. +# +# Requires curl and sed executables to be installed in the standard PATH. +# +# References +# https://pushover.net/api +# +# Arguments +# subject: notification subject +# pathname: pathname containing the notification message (OPTIONAL) +# +# Globals +# ZED_PUSHOVER_TOKEN +# ZED_PUSHOVER_USER +# +# Return +# 0: notification sent +# 1: notification failed +# 2: not configured +# +zed_notify_pushover() +{ + local subject="$1" + local pathname="${2:-"/dev/null"}" + local msg_body + local msg_out + local msg_err + local url="https://api.pushover.net/1/messages.json" + + [ -n "${ZED_PUSHOVER_TOKEN}" ] && [ -n "${ZED_PUSHOVER_USER}" ] || return 2 + + if [ ! -r "${pathname}" ]; then + zed_log_err "pushover cannot read \"${pathname}\"" + return 1 + fi + + zed_check_cmd "curl" "sed" || return 1 + + # Read the message body in. + # + msg_body="$(cat "${pathname}")" + + if [ -z "${msg_body}" ] + then + msg_body=$subject + subject="" + fi + + # Send the POST request and check for errors. + # + msg_out="$( \ + curl \ + --form-string "token=${ZED_PUSHOVER_TOKEN}" \ + --form-string "user=${ZED_PUSHOVER_USER}" \ + --form-string "message=${msg_body}" \ + --form-string "title=${subject}" \ + "${url}" \ + 2>/dev/null \ + )"; rv=$? + if [ "${rv}" -ne 0 ]; then + zed_log_err "curl exit=${rv}" + return 1 + fi + msg_err="$(echo "${msg_out}" \ + | sed -n -e 's/.*"errors" *:.*\[\(.*\)\].*/\1/p')" + if [ -n "${msg_err}" ]; then + zed_log_err "pushover \"${msg_err}"\" + return 1 + fi + return 0 +} + + +# zed_notify_ntfy (subject, pathname) +# +# Send a notification via Ntfy.sh <https://ntfy.sh/>. +# The ntfy topic (ZED_NTFY_TOPIC) identifies the topic that the notification +# will be sent to Ntfy.sh server. The ntfy url (ZED_NTFY_URL) defines the +# self-hosted or provided hosted ntfy service location. The ntfy access token +# <https://docs.ntfy.sh/publish/#access-tokens> (ZED_NTFY_ACCESS_TOKEN) reprsents an +# access token that could be used if a topic is read/write protected. If a +# topic can be written to publicaly, a ZED_NTFY_ACCESS_TOKEN is not required. +# +# Requires curl and sed executables to be installed in the standard PATH. +# +# References +# https://docs.ntfy.sh +# +# Arguments +# subject: notification subject +# pathname: pathname containing the notification message (OPTIONAL) +# +# Globals +# ZED_NTFY_TOPIC +# ZED_NTFY_ACCESS_TOKEN (OPTIONAL) +# ZED_NTFY_URL +# +# Return +# 0: notification sent +# 1: notification failed +# 2: not configured +# +zed_notify_ntfy() +{ + local subject="$1" + local pathname="${2:-"/dev/null"}" + local msg_body + local msg_out + local msg_err + + [ -n "${ZED_NTFY_TOPIC}" ] || return 2 + local url="${ZED_NTFY_URL:-"https://ntfy.sh"}/${ZED_NTFY_TOPIC}" + + if [ ! -r "${pathname}" ]; then + zed_log_err "ntfy cannot read \"${pathname}\"" + return 1 + fi + + zed_check_cmd "curl" "sed" || return 1 + + # Read the message body in. + # + msg_body="$(cat "${pathname}")" + + if [ -z "${msg_body}" ] + then + msg_body=$subject + subject="" + fi + + # Send the POST request and check for errors. + # + if [ -n "${ZED_NTFY_ACCESS_TOKEN}" ]; then + msg_out="$( \ + curl \ + -u ":${ZED_NTFY_ACCESS_TOKEN}" \ + -H "Title: ${subject}" \ + -d "${msg_body}" \ + -H "Priority: high" \ + "${url}" \ + 2>/dev/null \ + )"; rv=$? + else + msg_out="$( \ + curl \ + -H "Title: ${subject}" \ + -d "${msg_body}" \ + -H "Priority: high" \ + "${url}" \ + 2>/dev/null \ + )"; rv=$? + fi + if [ "${rv}" -ne 0 ]; then + zed_log_err "curl exit=${rv}" + return 1 + fi + msg_err="$(echo "${msg_out}" \ + | sed -n -e 's/.*"errors" *:.*\[\(.*\)\].*/\1/p')" + if [ -n "${msg_err}" ]; then + zed_log_err "ntfy \"${msg_err}"\" + return 1 + fi + return 0 +} + + +# zed_notify_gotify (subject, pathname) +# +# Send a notification via Gotify <https://gotify.net/>. +# The Gotify URL (ZED_GOTIFY_URL) defines a self-hosted Gotify location. +# The Gotify application token (ZED_GOTIFY_APPTOKEN) defines a +# Gotify application token which is associated with a message. +# The optional Gotify priority value (ZED_GOTIFY_PRIORITY) overrides the +# default or configured priority at the Gotify server for the application. +# +# Requires curl and sed executables to be installed in the standard PATH. +# +# References +# https://gotify.net/docs/index +# +# Arguments +# subject: notification subject +# pathname: pathname containing the notification message (OPTIONAL) +# +# Globals +# ZED_GOTIFY_URL +# ZED_GOTIFY_APPTOKEN +# ZED_GOTIFY_PRIORITY +# +# Return +# 0: notification sent +# 1: notification failed +# 2: not configured +# +zed_notify_gotify() +{ + local subject="$1" + local pathname="${2:-"/dev/null"}" + local msg_body + local msg_out + local msg_err + + [ -n "${ZED_GOTIFY_URL}" ] && [ -n "${ZED_GOTIFY_APPTOKEN}" ] || return 2 + local url="${ZED_GOTIFY_URL}/message?token=${ZED_GOTIFY_APPTOKEN}" + + if [ ! -r "${pathname}" ]; then + zed_log_err "gotify cannot read \"${pathname}\"" + return 1 + fi + + zed_check_cmd "curl" "sed" || return 1 + + # Read the message body in. + # + msg_body="$(cat "${pathname}")" + + if [ -z "${msg_body}" ] + then + msg_body=$subject + subject="" + fi + + # Send the POST request and check for errors. + # + if [ -n "${ZED_GOTIFY_PRIORITY}" ]; then + msg_out="$( \ + curl \ + --form-string "title=${subject}" \ + --form-string "message=${msg_body}" \ + --form-string "priority=${ZED_GOTIFY_PRIORITY}" \ + "${url}" \ + 2>/dev/null \ + )"; rv=$? + else + msg_out="$( \ + curl \ + --form-string "title=${subject}" \ + --form-string "message=${msg_body}" \ + "${url}" \ + 2>/dev/null \ + )"; rv=$? + fi + + if [ "${rv}" -ne 0 ]; then + zed_log_err "curl exit=${rv}" + return 1 + fi + msg_err="$(echo "${msg_out}" \ + | sed -n -e 's/.*"errors" *:.*\[\(.*\)\].*/\1/p')" + if [ -n "${msg_err}" ]; then + zed_log_err "gotify \"${msg_err}"\" + return 1 + fi + return 0 +} + + + +# zed_rate_limit (tag, [interval]) +# +# Check whether an event of a given type [tag] has already occurred within the +# last [interval] seconds. +# +# This function obtains a lock on the statefile using file descriptor 9. +# +# Arguments +# tag: arbitrary string for grouping related events to rate-limit +# interval: time interval in seconds (OPTIONAL) +# +# Globals +# ZED_NOTIFY_INTERVAL_SECS +# ZED_RUNDIR +# +# Return +# 0 if the event should be processed +# 1 if the event should be dropped +# +# State File Format +# time;tag +# +zed_rate_limit() +{ + local tag="$1" + local interval="${2:-${ZED_NOTIFY_INTERVAL_SECS}}" + local lockfile="zed.zedlet.state.lock" + local lockfile_fd=9 + local statefile="${ZED_RUNDIR}/zed.zedlet.state" + local time_now + local time_prev + local umask_bak + local rv=0 + + [ -n "${tag}" ] || return 0 + + zed_lock "${lockfile}" "${lockfile_fd}" + time_now="$(date +%s)" + time_prev="$(grep -E "^[0-9]+;${tag}\$" "${statefile}" 2>/dev/null \ + | tail -1 | cut -d\; -f1)" + + if [ -n "${time_prev}" ] \ + && [ "$((time_now - time_prev))" -lt "${interval}" ]; then + rv=1 + else + umask_bak="$(umask)" + umask 077 + grep -E -v "^[0-9]+;${tag}\$" "${statefile}" 2>/dev/null \ + > "${statefile}.$$" + echo "${time_now};${tag}" >> "${statefile}.$$" + mv -f "${statefile}.$$" "${statefile}" + umask "${umask_bak}" + fi + + zed_unlock "${lockfile}" "${lockfile_fd}" + return "${rv}" +} + + +# zed_guid_to_pool (guid) +# +# Convert a pool GUID into its pool name (like "tank") +# Arguments +# guid: pool GUID (decimal or hex) +# +# Return +# Pool name +# +zed_guid_to_pool() +{ + if [ -z "$1" ] ; then + return + fi + + guid="$(printf "%u" "$1")" + $ZPOOL get -H -ovalue,name guid | awk '$1 == '"$guid"' {print $2; exit}' +} + +# zed_exit_if_ignoring_this_event +# +# Exit the script if we should ignore this event, as determined by +# $ZED_SYSLOG_SUBCLASS_INCLUDE and $ZED_SYSLOG_SUBCLASS_EXCLUDE in zed.rc. +# This function assumes you've imported the normal zed variables. +zed_exit_if_ignoring_this_event() +{ + if [ -n "${ZED_SYSLOG_SUBCLASS_INCLUDE}" ]; then + eval "case ${ZEVENT_SUBCLASS} in + ${ZED_SYSLOG_SUBCLASS_INCLUDE});; + *) exit 0;; + esac" + elif [ -n "${ZED_SYSLOG_SUBCLASS_EXCLUDE}" ]; then + eval "case ${ZEVENT_SUBCLASS} in + ${ZED_SYSLOG_SUBCLASS_EXCLUDE}) exit 0;; + *);; + esac" + fi +} diff --git a/straper/db/public/zfs/etc-zfs/zed.d/zed.rc b/straper/db/public/zfs/etc-zfs/zed.d/zed.rc new file mode 100644 index 0000000..47fb016 --- /dev/null +++ b/straper/db/public/zfs/etc-zfs/zed.d/zed.rc @@ -0,0 +1,199 @@ +## +# zed.rc – ZEDLET configuration. +## +# shellcheck disable=SC2034 + +## +# Absolute path to the debug output file. +# +#ZED_DEBUG_LOG="/tmp/zed.debug.log" + +## +# Email address of the zpool administrator for receipt of notifications; +# multiple addresses can be specified if they are delimited by whitespace. +# Email will only be sent if ZED_EMAIL_ADDR is defined. +# Enabled by default; comment to disable. +# +ZED_EMAIL_ADDR="root" + +## +# Name or path of executable responsible for sending notifications via email; +# the mail program must be capable of reading a message body from stdin. +# Email will only be sent if ZED_EMAIL_ADDR is defined. +# +#ZED_EMAIL_PROG="mail" + +## +# Command-line options for ZED_EMAIL_PROG. +# The string @ADDRESS@ will be replaced with the recipient email address(es). +# The string @SUBJECT@ will be replaced with the notification subject; +# this should be protected with quotes to prevent word-splitting. +# Email will only be sent if ZED_EMAIL_ADDR is defined. +# If @SUBJECT@ was omited here, a "Subject: ..." header will be added to notification +# +#ZED_EMAIL_OPTS="-s '@SUBJECT@' @ADDRESS@" + +## +# Default directory for zed lock files. +# +#ZED_LOCKDIR="/var/lock" + +## +# Minimum number of seconds between notifications for a similar event. +# +ZED_NOTIFY_INTERVAL_SECS=3600 + +## +# Notification verbosity. +# If set to 0, suppress notification if the pool is healthy. +# If set to 1, send notification regardless of pool health. +# +#ZED_NOTIFY_VERBOSE=0 + +## +# Send notifications for 'ereport.fs.zfs.data' events. +# Disabled by default, any non-empty value will enable the feature. +# +#ZED_NOTIFY_DATA= + +## +# Pushbullet access token. +# This grants full access to your account -- protect it accordingly! +# <https://www.pushbullet.com/get-started> +# <https://www.pushbullet.com/account> +# Disabled by default; uncomment to enable. +# +#ZED_PUSHBULLET_ACCESS_TOKEN="" + +## +# Pushbullet channel tag for push notification feeds that can be subscribed to. +# <https://www.pushbullet.com/my-channel> +# If not defined, push notifications will instead be sent to all devices +# associated with the account specified by the access token. +# Disabled by default; uncomment to enable. +# +#ZED_PUSHBULLET_CHANNEL_TAG="" + +## +# Slack Webhook URL. +# This allows posting to the given channel and includes an access token. +# <https://api.slack.com/incoming-webhooks> +# Disabled by default; uncomment to enable. +# +#ZED_SLACK_WEBHOOK_URL="" + +## +# Pushover token. +# This defines the application from which the notification will be sent. +# <https://pushover.net/api#registration> +# Disabled by default; uncomment to enable. +# ZED_PUSHOVER_USER, below, must also be configured. +# +#ZED_PUSHOVER_TOKEN="" + +## +# Pushover user key. +# This defines which user or group will receive Pushover notifications. +# <https://pushover.net/api#identifiers> +# Disabled by default; uncomment to enable. +# ZED_PUSHOVER_TOKEN, above, must also be configured. +#ZED_PUSHOVER_USER="" + +## +# Default directory for zed state files. +# +#ZED_RUNDIR="/var/run" + +## +# Turn on/off enclosure LEDs when drives get DEGRADED/FAULTED. This works for +# device mapper and multipath devices as well. This works with JBOD enclosures +# and NVMe PCI drives (assuming they're supported by Linux in sysfs). +# +ZED_USE_ENCLOSURE_LEDS=1 + +## +# Run a scrub after every resilver +# Disabled by default, 1 to enable and 0 to disable. +#ZED_SCRUB_AFTER_RESILVER=0 + +## +# The syslog priority (e.g., specified as a "facility.level" pair). +# +#ZED_SYSLOG_PRIORITY="daemon.notice" + +## +# The syslog tag for marking zed events. +# +#ZED_SYSLOG_TAG="zed" + +## +# Which set of event subclasses to log +# By default, events from all subclasses are logged. +# If ZED_SYSLOG_SUBCLASS_INCLUDE is set, only subclasses +# matching the pattern are logged. Use the pipe symbol (|) +# or shell wildcards (*, ?) to match multiple subclasses. +# Otherwise, if ZED_SYSLOG_SUBCLASS_EXCLUDE is set, the +# matching subclasses are excluded from logging. +#ZED_SYSLOG_SUBCLASS_INCLUDE="checksum|scrub_*|vdev.*" +ZED_SYSLOG_SUBCLASS_EXCLUDE="history_event" + +## +# Use GUIDs instead of names when logging pool and vdevs +# Disabled by default, 1 to enable and 0 to disable. +#ZED_SYSLOG_DISPLAY_GUIDS=1 + +## +# Power off the drive's slot in the enclosure if it becomes FAULTED. This can +# help silence misbehaving drives. This assumes your drive enclosure fully +# supports slot power control via sysfs. +#ZED_POWER_OFF_ENCLOSURE_SLOT_ON_FAULT=1 + +## +# Power off the drive's slot in the enclosure if there is a hung I/O which +# exceeds the deadman timeout. This can help prevent a single misbehaving +# drive from rendering a redundant pool unavailable. This assumes your drive +# enclosure fully supports slot power control via sysfs. +#ZED_POWER_OFF_ENCLOSURE_SLOT_ON_DEADMAN=1 + +## +# Ntfy topic +# This defines which topic will receive the ntfy notification. +# <https://docs.ntfy.sh/publish/> +# Disabled by default; uncomment to enable. +#ZED_NTFY_TOPIC="" + +## +# Ntfy access token (optional for public topics) +# This defines an access token which can be used +# to allow you to authenticate when sending to topics +# <https://docs.ntfy.sh/publish/#access-tokens> +# Disabled by default; uncomment to enable. +#ZED_NTFY_ACCESS_TOKEN="" + +## +# Ntfy Service URL +# This defines which service the ntfy call will be directed toward +# <https://docs.ntfy.sh/install/> +# https://ntfy.sh by default; uncomment to enable an alternative service url. +#ZED_NTFY_URL="https://ntfy.sh" + +## +# Gotify server URL +# This defines a URL that the Gotify call will be directed toward. +# <https://gotify.net/docs/index> +# Disabled by default; uncomment to enable. +#ZED_GOTIFY_URL="" + +## +# Gotify application token +# This defines a Gotify application token which a message is associated with. +# This token is generated when an application is created on the Gotify server. +# Disabled by default; uncomment to enable. +#ZED_GOTIFY_APPTOKEN="" + +## +# Gotify priority (optional) +# If defined, this overrides the default priority of the +# Gotify application associated with ZED_GOTIFY_APPTOKEN. +# Value is an integer 0 and up. +#ZED_GOTIFY_PRIORITY="" |