feat: first fully successful run e2e on straper

15 files changed, 2524 insertions, 0 deletions

diff --git a/straper/db/public/zfs/etc-zfs/zed.d/all-syslog.sh b/straper/db/public/zfs/etc-zfs/zed.d/all-syslog.sh
new file mode 100755
index 0000000..5f60114
--- /dev/null
+++ b/straper/db/public/zfs/etc-zfs/zed.d/all-syslog.sh
@@ -0,0 +1,52 @@
+#!/bin/sh
+# shellcheck disable=SC2154
+#
+# Copyright (C) 2013-2014 Lawrence Livermore National Security, LLC.
+# Copyright (c) 2020 by Delphix. All rights reserved.
+#
+
+#
+# Log the zevent via syslog.
+#
+
+[ -f "${ZED_ZEDLET_DIR}/zed.rc" ] && . "${ZED_ZEDLET_DIR}/zed.rc"
+. "${ZED_ZEDLET_DIR}/zed-functions.sh"
+
+zed_exit_if_ignoring_this_event
+
+# build a string of name=value pairs for this event
+msg="eid=${ZEVENT_EID} class=${ZEVENT_SUBCLASS}"
+
+if [ "${ZED_SYSLOG_DISPLAY_GUIDS}" = "1" ]; then
+    [ -n "${ZEVENT_POOL_GUID}" ] && msg="${msg} pool_guid=${ZEVENT_POOL_GUID}"
+    [ -n "${ZEVENT_VDEV_GUID}" ] && msg="${msg} vdev_guid=${ZEVENT_VDEV_GUID}"
+else
+    [ -n "${ZEVENT_POOL}" ] && msg="${msg} pool='${ZEVENT_POOL}'"
+    [ -n "${ZEVENT_VDEV_PATH}" ] && msg="${msg} vdev=${ZEVENT_VDEV_PATH##*/}"
+fi
+
+# log pool state if state is anything other than 'ACTIVE'
+[ -n "${ZEVENT_POOL_STATE_STR}" ] && [ "$ZEVENT_POOL_STATE" -ne 0 ] && \
+    msg="${msg} pool_state=${ZEVENT_POOL_STATE_STR}"
+
+# Log the following payload nvpairs if they are present
+[ -n "${ZEVENT_VDEV_STATE_STR}" ]  && msg="${msg} vdev_state=${ZEVENT_VDEV_STATE_STR}"
+[ -n "${ZEVENT_CKSUM_ALGORITHM}" ] && msg="${msg} algorithm=${ZEVENT_CKSUM_ALGORITHM}"
+[ -n "${ZEVENT_ZIO_SIZE}" ]        && msg="${msg} size=${ZEVENT_ZIO_SIZE}"
+[ -n "${ZEVENT_ZIO_OFFSET}" ]      && msg="${msg} offset=${ZEVENT_ZIO_OFFSET}"
+[ -n "${ZEVENT_ZIO_PRIORITY}" ]    && msg="${msg} priority=${ZEVENT_ZIO_PRIORITY}"
+[ -n "${ZEVENT_ZIO_ERR}" ]         && msg="${msg} err=${ZEVENT_ZIO_ERR}"
+[ -n "${ZEVENT_ZIO_FLAGS}" ]       && msg="${msg} flags=$(printf '0x%x' "${ZEVENT_ZIO_FLAGS}")"
+
+# log delays that are >= 10 milisec
+[ -n "${ZEVENT_ZIO_DELAY}" ] && [ "$ZEVENT_ZIO_DELAY" -gt 10000000 ] && \
+    msg="${msg} delay=$((ZEVENT_ZIO_DELAY / 1000000))ms"
+
+# list the bookmark data together
+# shellcheck disable=SC2153
+[ -n "${ZEVENT_ZIO_OBJSET}" ] && \
+    msg="${msg} bookmark=${ZEVENT_ZIO_OBJSET}:${ZEVENT_ZIO_OBJECT}:${ZEVENT_ZIO_LEVEL}:${ZEVENT_ZIO_BLKID}"
+
+zed_log_msg "${msg}"
+
+exit 0
diff --git a/straper/db/public/zfs/etc-zfs/zed.d/data-notify.sh b/straper/db/public/zfs/etc-zfs/zed.d/data-notify.sh
new file mode 100755
index 0000000..9846769
--- /dev/null
+++ b/straper/db/public/zfs/etc-zfs/zed.d/data-notify.sh
@@ -0,0 +1,44 @@
+#!/bin/sh
+# shellcheck disable=SC2154
+#
+# Send notification in response to a DATA error.
+#
+# Only one notification per ZED_NOTIFY_INTERVAL_SECS will be sent for a given
+# class/pool/[vdev] combination.  This protects against spamming the recipient
+# should multiple events occur together in time for the same pool/[vdev].
+#
+# Exit codes:
+#   0: notification sent
+#   1: notification failed
+#   2: notification not configured
+#   3: notification suppressed
+#   9: internal error
+
+[ -f "${ZED_ZEDLET_DIR}/zed.rc" ] && . "${ZED_ZEDLET_DIR}/zed.rc"
+. "${ZED_ZEDLET_DIR}/zed-functions.sh"
+
+[ -n "${ZEVENT_POOL}" ] || exit 9
+[ -n "${ZEVENT_SUBCLASS}" ] || exit 9
+[ -n "${ZED_NOTIFY_DATA}" ] || exit 3
+
+rate_limit_tag="${ZEVENT_POOL};${ZEVENT_VDEV_GUID:-0};${ZEVENT_SUBCLASS};notify"
+zed_rate_limit "${rate_limit_tag}" || exit 3
+
+umask 077
+note_subject="ZFS ${ZEVENT_SUBCLASS} error for ${ZEVENT_POOL} on $(hostname)"
+note_pathname="$(mktemp)"
+{
+    echo "ZFS has detected a data error:"
+    echo
+    echo "   eid: ${ZEVENT_EID}"
+    echo " class: ${ZEVENT_SUBCLASS}"
+    echo "  host: $(hostname)"
+    echo "  time: ${ZEVENT_TIME_STRING}"
+    echo " error: ${ZEVENT_ZIO_ERR}"
+    echo " objid: ${ZEVENT_ZIO_OBJSET}:${ZEVENT_ZIO_OBJECT}"
+    echo "  pool: ${ZEVENT_POOL}"
+} > "${note_pathname}"
+
+zed_notify "${note_subject}" "${note_pathname}"; rv=$?
+rm -f "${note_pathname}"
+exit "${rv}"
diff --git a/straper/db/public/zfs/etc-zfs/zed.d/deadman-slot_off.sh b/straper/db/public/zfs/etc-zfs/zed.d/deadman-slot_off.sh
new file mode 100755
index 0000000..7b339b3
--- /dev/null
+++ b/straper/db/public/zfs/etc-zfs/zed.d/deadman-slot_off.sh
@@ -0,0 +1,71 @@
+#!/bin/sh
+# shellcheck disable=SC3014,SC2154,SC2086,SC2034
+#
+# Turn off disk's enclosure slot if an I/O is hung triggering the deadman.
+#
+# It's possible for outstanding I/O to a misbehaving SCSI disk to neither
+# promptly complete or return an error.  This can occur due to retry and
+# recovery actions taken by the SCSI layer, driver, or disk.  When it occurs
+# the pool will be unresponsive even though there may be sufficient redundancy
+# configured to proceeded without this single disk.
+#
+# When a hung I/O is detected by the kmods it will be posted as a deadman
+# event.  By default an I/O is considered to be hung after 5 minutes.  This
+# value can be changed with the zfs_deadman_ziotime_ms module parameter.
+# If ZED_POWER_OFF_ENCLOSURE_SLOT_ON_DEADMAN is set the disk's enclosure
+# slot will be powered off causing the outstanding I/O to fail.  The ZED
+# will then handle this like a normal disk failure and FAULT the vdev.
+#
+# We assume the user will be responsible for turning the slot back on
+# after replacing the disk.
+#
+# Note that this script requires that your enclosure be supported by the
+# Linux SCSI Enclosure services (SES) driver.  The script will do nothing
+# if you have no enclosure, or if your enclosure isn't supported.
+#
+# Exit codes:
+#   0: slot successfully powered off
+#   1: enclosure not available
+#   2: ZED_POWER_OFF_ENCLOSURE_SLOT_ON_DEADMAN disabled
+#   3: System not configured to wait on deadman
+#   4: The enclosure sysfs path passed from ZFS does not exist
+#   5: Enclosure slot didn't actually turn off after we told it to
+
+[ -f "${ZED_ZEDLET_DIR}/zed.rc" ] && . "${ZED_ZEDLET_DIR}/zed.rc"
+. "${ZED_ZEDLET_DIR}/zed-functions.sh"
+
+if [ ! -d /sys/class/enclosure ] ; then
+	# No JBOD enclosure or NVMe slots
+	exit 1
+fi
+
+if [ "${ZED_POWER_OFF_ENCLOSURE_SLOT_ON_DEADMAN}" != "1" ] ; then
+	exit 2
+fi
+
+if [ "$ZEVENT_POOL_FAILMODE" != "wait" ] ; then
+	exit 3
+fi
+
+if [ ! -f "$ZEVENT_VDEV_ENC_SYSFS_PATH/power_status" ] ; then
+	exit 4
+fi
+
+# Turn off the slot and wait for sysfs to report that the slot is off.
+# It can take ~400ms on some enclosures and multiple retries may be needed.
+for i in $(seq 1 20) ; do
+	echo "off" | tee "$ZEVENT_VDEV_ENC_SYSFS_PATH/power_status"
+
+	for j in $(seq 1 5) ; do
+		if [ "$(cat $ZEVENT_VDEV_ENC_SYSFS_PATH/power_status)" == "off" ] ; then
+			break 2
+		fi
+		sleep 0.1
+	done
+done
+
+if [ "$(cat $ZEVENT_VDEV_ENC_SYSFS_PATH/power_status)" != "off" ] ; then
+	exit 5
+fi
+
+zed_log_msg "powered down slot $ZEVENT_VDEV_ENC_SYSFS_PATH for $ZEVENT_VDEV_PATH"
diff --git a/straper/db/public/zfs/etc-zfs/zed.d/history_event-zfs-list-cacher.sh b/straper/db/public/zfs/etc-zfs/zed.d/history_event-zfs-list-cacher.sh
new file mode 100755
index 0000000..cacf921
--- /dev/null
+++ b/straper/db/public/zfs/etc-zfs/zed.d/history_event-zfs-list-cacher.sh
@@ -0,0 +1,85 @@
+#!/bin/sh
+# shellcheck disable=SC2154
+#
+# Track changes to enumerated pools for use in early-boot
+set -ef
+
+FSLIST="/etc/zfs/zfs-list.cache/${ZEVENT_POOL}"
+FSLIST_TMP="/run/zfs-list.cache@${ZEVENT_POOL}"
+
+# If the pool specific cache file is not writeable, abort
+[ -w "${FSLIST}" ] || exit 0
+
+[ -f "${ZED_ZEDLET_DIR}/zed.rc" ] && . "${ZED_ZEDLET_DIR}/zed.rc"
+. "${ZED_ZEDLET_DIR}/zed-functions.sh"
+
+[ "$ZEVENT_SUBCLASS" != "history_event" ] && exit 0
+zed_check_cmd "${ZFS}" sort diff
+
+# If we are acting on a snapshot, we have nothing to do
+[ "${ZEVENT_HISTORY_DSNAME%@*}" = "${ZEVENT_HISTORY_DSNAME}" ] || exit 0
+
+# We lock the output file to avoid simultaneous writes.
+# If we run into trouble, log and drop the lock
+abort_alter() {
+  zed_log_msg "Error updating zfs-list.cache for ${ZEVENT_POOL}!"
+  zed_unlock "${FSLIST}"
+}
+
+finished() {
+  zed_unlock "${FSLIST}"
+  trap - EXIT
+  exit 0
+}
+
+case "${ZEVENT_HISTORY_INTERNAL_NAME}" in
+    create|"finish receiving"|import|destroy|rename)
+      ;;
+
+    export)
+        zed_lock "${FSLIST}"
+        trap abort_alter EXIT
+        echo > "${FSLIST}"
+        finished
+      ;;
+
+    set|inherit)
+        # Only act if one of the tracked properties is altered.
+        case "${ZEVENT_HISTORY_INTERNAL_STR%%=*}" in
+            canmount|mountpoint|atime|relatime|devices|exec|readonly| \
+              setuid|nbmand|encroot|keylocation|org.openzfs.systemd:requires| \
+              org.openzfs.systemd:requires-mounts-for| \
+              org.openzfs.systemd:before|org.openzfs.systemd:after| \
+              org.openzfs.systemd:wanted-by|org.openzfs.systemd:required-by| \
+              org.openzfs.systemd:nofail|org.openzfs.systemd:ignore \
+            ) ;;
+            *) exit 0 ;;
+        esac
+      ;;
+
+    *)
+        # Ignore all other events.
+        exit 0
+      ;;
+esac
+
+zed_lock "${FSLIST}"
+trap abort_alter EXIT
+
+PROPS="name,mountpoint,canmount,atime,relatime,devices,exec\
+,readonly,setuid,nbmand,encroot,keylocation\
+,org.openzfs.systemd:requires,org.openzfs.systemd:requires-mounts-for\
+,org.openzfs.systemd:before,org.openzfs.systemd:after\
+,org.openzfs.systemd:wanted-by,org.openzfs.systemd:required-by\
+,org.openzfs.systemd:nofail,org.openzfs.systemd:ignore"
+
+"${ZFS}" list -H -t filesystem -o "${PROPS}" -r "${ZEVENT_POOL}" > "${FSLIST_TMP}"
+
+# Sort the output so that it is stable
+sort "${FSLIST_TMP}" -o "${FSLIST_TMP}"
+
+# Don't modify the file if it hasn't changed
+diff -q "${FSLIST_TMP}" "${FSLIST}" || cat "${FSLIST_TMP}" > "${FSLIST}"
+rm -f "${FSLIST_TMP}"
+
+finished
diff --git a/straper/db/public/zfs/etc-zfs/zed.d/pool_import-led.sh b/straper/db/public/zfs/etc-zfs/zed.d/pool_import-led.sh
new file mode 100755
index 0000000..40cb61f
--- /dev/null
+++ b/straper/db/public/zfs/etc-zfs/zed.d/pool_import-led.sh
@@ -0,0 +1,242 @@
+#!/bin/sh
+# shellcheck disable=SC2154
+#
+# Turn off/on vdevs' enclosure fault LEDs when their pool's state changes.
+#
+# Turn a vdev's fault LED on if it becomes FAULTED, DEGRADED or UNAVAIL.
+# Turn its LED off when it's back ONLINE again.
+#
+# This script run in two basic modes:
+#
+# 1. If $ZEVENT_VDEV_ENC_SYSFS_PATH and $ZEVENT_VDEV_STATE_STR are set, then
+# only set the LED for that particular vdev. This is the case for statechange
+# events and some vdev_* events.
+#
+# 2. If those vars are not set, then check the state of all vdevs in the pool
+# and set the LEDs accordingly.  This is the case for pool_import events.
+#
+# Note that this script requires that your enclosure be supported by the
+# Linux SCSI Enclosure services (SES) driver.  The script will do nothing
+# if you have no enclosure, or if your enclosure isn't supported.
+#
+# Exit codes:
+#   0: enclosure led successfully set
+#   1: enclosure leds not available
+#   2: enclosure leds administratively disabled
+#   3: The led sysfs path passed from ZFS does not exist
+#   4: $ZPOOL not set
+#   5: awk is not installed
+
+[ -f "${ZED_ZEDLET_DIR}/zed.rc" ] && . "${ZED_ZEDLET_DIR}/zed.rc"
+. "${ZED_ZEDLET_DIR}/zed-functions.sh"
+
+if [ ! -d /sys/class/enclosure ] && [ ! -d /sys/bus/pci/slots ] ; then
+	# No JBOD enclosure or NVMe slots
+	exit 1
+fi
+
+if [ "${ZED_USE_ENCLOSURE_LEDS}" != "1" ] ; then
+	exit 2
+fi
+
+zed_check_cmd "$ZPOOL" || exit 4
+zed_check_cmd awk || exit 5
+
+# Global used in set_led debug print
+vdev=""
+
+# check_and_set_led (file, val)
+#
+# Read an enclosure sysfs file, and write it if it's not already set to 'val'
+#
+# Arguments
+#   file: sysfs file to set (like /sys/class/enclosure/0:0:1:0/SLOT 10/fault)
+#   val: value to set it to
+#
+# Return
+#  0 on success, 3 on missing sysfs path
+#
+check_and_set_led()
+{
+	file="$1"
+	val="$2"
+
+	if [ -z "$val" ]; then
+		return 0
+	fi
+
+	if [ ! -e "$file" ] ; then
+		return 3
+	fi
+
+	# If another process is accessing the LED when we attempt to update it,
+	# the update will be lost so retry until the LED actually changes or we
+	# timeout.
+	for _ in 1 2 3 4 5; do
+		# We want to check the current state first, since writing to the
+		# 'fault' entry always causes a SES command, even if the
+		# current state is already what you want.
+		read -r current < "${file}"
+
+		# On some enclosures if you write 1 to fault, and read it back,
+		# it will return 2.  Treat all non-zero values as 1 for
+		# simplicity.
+		if [ "$current" != "0" ] ; then
+			current=1
+		fi
+
+		if [ "$current" != "$val" ] ; then
+			echo "$val" > "$file"
+			zed_log_msg "vdev $vdev set '$file' LED to $val"
+		else
+			break
+		fi
+	done
+}
+
+# Fault LEDs for JBODs and NVMe drives are handled a little differently.
+#
+# On JBODs the fault LED is called 'fault' and on a path like this:
+#
+#   /sys/class/enclosure/0:0:1:0/SLOT 10/fault
+#
+# On NVMe it's called 'attention' and on a path like this:
+#
+#   /sys/bus/pci/slot/0/attention
+#
+# This function returns the full path to the fault LED file for a given
+# enclosure/slot directory.
+#
+path_to_led()
+{
+	dir=$1
+	if [ -f "$dir/fault" ] ; then
+		echo "$dir/fault"
+	elif [ -f "$dir/attention" ] ; then
+		echo "$dir/attention"
+	fi
+}
+
+state_to_val()
+{
+	state="$1"
+	case "$state" in
+		FAULTED|DEGRADED|UNAVAIL|REMOVED)
+			echo 1
+			;;
+		ONLINE)
+			echo 0
+			;;
+		*)
+			echo "invalid state: $state"
+			;;
+	esac
+}
+
+#
+# Given a nvme name like 'nvme0n1', pass back its slot directory
+# like "/sys/bus/pci/slots/0"
+#
+nvme_dev_to_slot()
+{
+	dev="$1"
+
+	# Get the address "0000:01:00.0"
+	read -r address < "/sys/class/block/$dev/device/address"
+
+	find /sys/bus/pci/slots -regex '.*/[0-9]+/address$' | \
+		while read -r sys_addr; do
+			read -r this_address < "$sys_addr"
+
+			# The format of address is a little different between
+			# /sys/class/block/$dev/device/address and
+			# /sys/bus/pci/slots/
+			#
+			# address=           "0000:01:00.0"
+			# this_address =     "0000:01:00"
+			#
+			if echo "$address" | grep -Eq ^"$this_address" ; then
+				echo "${sys_addr%/*}"
+				break
+			fi
+			done
+}
+
+
+# process_pool (pool)
+#
+# Iterate through a pool and set the vdevs' enclosure slot LEDs to
+# those vdevs' state.
+#
+# Arguments
+#   pool:	Pool name.
+#
+# Return
+#  0 on success, 3 on missing sysfs path
+#
+process_pool()
+{
+	pool="$1"
+
+	# The output will be the vdevs only (from "grep '/dev/'"):
+	#
+	#    U45     ONLINE       0     0     0   /dev/sdk          0
+	#    U46     ONLINE       0     0     0   /dev/sdm          0
+	#    U47     ONLINE       0     0     0   /dev/sdn          0
+	#    U50     ONLINE       0     0     0  /dev/sdbn          0
+	#
+	ZPOOL_SCRIPTS_AS_ROOT=1 $ZPOOL status -c upath,fault_led "$pool" | grep '/dev/' | (
+	rc=0
+	while read -r vdev state _ _ _ therest; do
+		# Read out current LED value and path
+		# Get dev name (like 'sda')
+		dev=$(basename "$(echo "$therest" | awk '{print $(NF-1)}')")
+		vdev_enc_sysfs_path=$(realpath "/sys/class/block/$dev/device/enclosure_device"*)
+		if [ ! -d "$vdev_enc_sysfs_path" ] ; then
+			# This is not a JBOD disk, but it could be a PCI NVMe drive
+			vdev_enc_sysfs_path=$(nvme_dev_to_slot "$dev")
+		fi
+
+		current_val=$(echo "$therest" | awk '{print $NF}')
+
+		if [ "$current_val" != "0" ] ; then
+			current_val=1
+		fi
+
+		if [ -z "$vdev_enc_sysfs_path" ] ; then
+			# Skip anything with no sysfs LED entries
+			continue
+		fi
+
+		led_path=$(path_to_led "$vdev_enc_sysfs_path")
+		if [ ! -e "$led_path" ] ; then
+			rc=3
+			zed_log_msg "vdev $vdev '$led_path' doesn't exist"
+			continue
+		fi
+
+		val=$(state_to_val "$state")
+
+		if [ "$current_val" = "$val" ] ; then
+			# LED is already set correctly
+			continue
+		fi
+
+		if ! check_and_set_led "$led_path" "$val"; then
+			rc=3
+		fi
+	done
+	exit "$rc"; )
+}
+
+if [ -n "$ZEVENT_VDEV_ENC_SYSFS_PATH" ] && [ -n "$ZEVENT_VDEV_STATE_STR" ] ; then
+	# Got a statechange for an individual vdev
+	val=$(state_to_val "$ZEVENT_VDEV_STATE_STR")
+	vdev=$(basename "$ZEVENT_VDEV_PATH")
+	ledpath=$(path_to_led "$ZEVENT_VDEV_ENC_SYSFS_PATH")
+	check_and_set_led "$ledpath" "$val"
+else
+	# Process the entire pool
+	poolname=$(zed_guid_to_pool "$ZEVENT_POOL_GUID")
+	process_pool "$poolname"
+fi
diff --git a/straper/db/public/zfs/etc-zfs/zed.d/resilver_finish-notify.sh b/straper/db/public/zfs/etc-zfs/zed.d/resilver_finish-notify.sh
new file mode 100755
index 0000000..fc9dc23
--- /dev/null
+++ b/straper/db/public/zfs/etc-zfs/zed.d/resilver_finish-notify.sh
@@ -0,0 +1,60 @@
+#!/bin/sh
+# shellcheck disable=SC2154
+#
+# Send notification in response to a RESILVER_FINISH or SCRUB_FINISH.
+#
+# By default, "zpool status" output will only be included for a scrub_finish
+# zevent if the pool is not healthy; to always include its output, set
+# ZED_NOTIFY_VERBOSE=1.
+#
+# Exit codes:
+#   0: notification sent
+#   1: notification failed
+#   2: notification not configured
+#   3: notification suppressed
+#   9: internal error
+
+[ -f "${ZED_ZEDLET_DIR}/zed.rc" ] && . "${ZED_ZEDLET_DIR}/zed.rc"
+. "${ZED_ZEDLET_DIR}/zed-functions.sh"
+
+[ -n "${ZEVENT_POOL}" ] || exit 9
+[ -n "${ZEVENT_SUBCLASS}" ] || exit 9
+
+if   [ "${ZEVENT_SUBCLASS}" = "resilver_finish" ]; then
+    action="resilver"
+elif [ "${ZEVENT_SUBCLASS}" = "scrub_finish" ]; then
+    action="scrub"
+else
+    zed_log_err "unsupported event class \"${ZEVENT_SUBCLASS}\""
+    exit 9
+fi
+
+zed_check_cmd "${ZPOOL}" || exit 9
+
+# For scrub, suppress notification if the pool is healthy
+# and verbosity is not enabled.
+#
+if [ "${ZEVENT_SUBCLASS}" = "scrub_finish" ]; then
+    healthy="$("${ZPOOL}" status -x "${ZEVENT_POOL}" \
+        | grep "'${ZEVENT_POOL}' is healthy")"
+    [ -n "${healthy}" ] && [ "${ZED_NOTIFY_VERBOSE}" -eq 0 ] && exit 3
+fi
+
+umask 077
+note_subject="ZFS ${ZEVENT_SUBCLASS} event for ${ZEVENT_POOL} on $(hostname)"
+note_pathname="$(mktemp)"
+{
+    echo "ZFS has finished a ${action}:"
+    echo
+    echo "   eid: ${ZEVENT_EID}"
+    echo " class: ${ZEVENT_SUBCLASS}"
+    echo "  host: $(hostname)"
+    echo "  time: ${ZEVENT_TIME_STRING}"
+
+    "${ZPOOL}" status "${ZEVENT_POOL}"
+
+} > "${note_pathname}"
+
+zed_notify "${note_subject}" "${note_pathname}"; rv=$?
+rm -f "${note_pathname}"
+exit "${rv}"
diff --git a/straper/db/public/zfs/etc-zfs/zed.d/resilver_finish-start-scrub.sh b/straper/db/public/zfs/etc-zfs/zed.d/resilver_finish-start-scrub.sh
new file mode 100755
index 0000000..cafce6f
--- /dev/null
+++ b/straper/db/public/zfs/etc-zfs/zed.d/resilver_finish-start-scrub.sh
@@ -0,0 +1,20 @@
+#!/bin/sh
+# shellcheck disable=SC2154
+# resilver_finish-start-scrub.sh
+# Run a scrub after a resilver
+#
+# Exit codes:
+# 1: Internal error
+# 2: Script wasn't enabled in zed.rc
+# 3: Scrubs are automatically started for sequential resilvers
+[ -f "${ZED_ZEDLET_DIR}/zed.rc" ] && . "${ZED_ZEDLET_DIR}/zed.rc"
+. "${ZED_ZEDLET_DIR}/zed-functions.sh"
+
+[ "${ZED_SCRUB_AFTER_RESILVER}" = "1" ] || exit 2
+[ "${ZEVENT_RESILVER_TYPE}" != "sequential" ] || exit 3
+[ -n "${ZEVENT_POOL}" ] || exit 1
+[ -n "${ZEVENT_SUBCLASS}" ] || exit 1
+zed_check_cmd "${ZPOOL}" || exit 1
+
+zed_log_msg "Starting scrub after resilver on ${ZEVENT_POOL}"
+"${ZPOOL}" scrub "${ZEVENT_POOL}"
diff --git a/straper/db/public/zfs/etc-zfs/zed.d/scrub_finish-notify.sh b/straper/db/public/zfs/etc-zfs/zed.d/scrub_finish-notify.sh
new file mode 100755
index 0000000..fc9dc23
--- /dev/null
+++ b/straper/db/public/zfs/etc-zfs/zed.d/scrub_finish-notify.sh
@@ -0,0 +1,60 @@
+#!/bin/sh
+# shellcheck disable=SC2154
+#
+# Send notification in response to a RESILVER_FINISH or SCRUB_FINISH.
+#
+# By default, "zpool status" output will only be included for a scrub_finish
+# zevent if the pool is not healthy; to always include its output, set
+# ZED_NOTIFY_VERBOSE=1.
+#
+# Exit codes:
+#   0: notification sent
+#   1: notification failed
+#   2: notification not configured
+#   3: notification suppressed
+#   9: internal error
+
+[ -f "${ZED_ZEDLET_DIR}/zed.rc" ] && . "${ZED_ZEDLET_DIR}/zed.rc"
+. "${ZED_ZEDLET_DIR}/zed-functions.sh"
+
+[ -n "${ZEVENT_POOL}" ] || exit 9
+[ -n "${ZEVENT_SUBCLASS}" ] || exit 9
+
+if   [ "${ZEVENT_SUBCLASS}" = "resilver_finish" ]; then
+    action="resilver"
+elif [ "${ZEVENT_SUBCLASS}" = "scrub_finish" ]; then
+    action="scrub"
+else
+    zed_log_err "unsupported event class \"${ZEVENT_SUBCLASS}\""
+    exit 9
+fi
+
+zed_check_cmd "${ZPOOL}" || exit 9
+
+# For scrub, suppress notification if the pool is healthy
+# and verbosity is not enabled.
+#
+if [ "${ZEVENT_SUBCLASS}" = "scrub_finish" ]; then
+    healthy="$("${ZPOOL}" status -x "${ZEVENT_POOL}" \
+        | grep "'${ZEVENT_POOL}' is healthy")"
+    [ -n "${healthy}" ] && [ "${ZED_NOTIFY_VERBOSE}" -eq 0 ] && exit 3
+fi
+
+umask 077
+note_subject="ZFS ${ZEVENT_SUBCLASS} event for ${ZEVENT_POOL} on $(hostname)"
+note_pathname="$(mktemp)"
+{
+    echo "ZFS has finished a ${action}:"
+    echo
+    echo "   eid: ${ZEVENT_EID}"
+    echo " class: ${ZEVENT_SUBCLASS}"
+    echo "  host: $(hostname)"
+    echo "  time: ${ZEVENT_TIME_STRING}"
+
+    "${ZPOOL}" status "${ZEVENT_POOL}"
+
+} > "${note_pathname}"
+
+zed_notify "${note_subject}" "${note_pathname}"; rv=$?
+rm -f "${note_pathname}"
+exit "${rv}"
diff --git a/straper/db/public/zfs/etc-zfs/zed.d/statechange-led.sh b/straper/db/public/zfs/etc-zfs/zed.d/statechange-led.sh
new file mode 100755
index 0000000..40cb61f
--- /dev/null
+++ b/straper/db/public/zfs/etc-zfs/zed.d/statechange-led.sh
@@ -0,0 +1,242 @@
+#!/bin/sh
+# shellcheck disable=SC2154
+#
+# Turn off/on vdevs' enclosure fault LEDs when their pool's state changes.
+#
+# Turn a vdev's fault LED on if it becomes FAULTED, DEGRADED or UNAVAIL.
+# Turn its LED off when it's back ONLINE again.
+#
+# This script run in two basic modes:
+#
+# 1. If $ZEVENT_VDEV_ENC_SYSFS_PATH and $ZEVENT_VDEV_STATE_STR are set, then
+# only set the LED for that particular vdev. This is the case for statechange
+# events and some vdev_* events.
+#
+# 2. If those vars are not set, then check the state of all vdevs in the pool
+# and set the LEDs accordingly.  This is the case for pool_import events.
+#
+# Note that this script requires that your enclosure be supported by the
+# Linux SCSI Enclosure services (SES) driver.  The script will do nothing
+# if you have no enclosure, or if your enclosure isn't supported.
+#
+# Exit codes:
+#   0: enclosure led successfully set
+#   1: enclosure leds not available
+#   2: enclosure leds administratively disabled
+#   3: The led sysfs path passed from ZFS does not exist
+#   4: $ZPOOL not set
+#   5: awk is not installed
+
+[ -f "${ZED_ZEDLET_DIR}/zed.rc" ] && . "${ZED_ZEDLET_DIR}/zed.rc"
+. "${ZED_ZEDLET_DIR}/zed-functions.sh"
+
+if [ ! -d /sys/class/enclosure ] && [ ! -d /sys/bus/pci/slots ] ; then
+	# No JBOD enclosure or NVMe slots
+	exit 1
+fi
+
+if [ "${ZED_USE_ENCLOSURE_LEDS}" != "1" ] ; then
+	exit 2
+fi
+
+zed_check_cmd "$ZPOOL" || exit 4
+zed_check_cmd awk || exit 5
+
+# Global used in set_led debug print
+vdev=""
+
+# check_and_set_led (file, val)
+#
+# Read an enclosure sysfs file, and write it if it's not already set to 'val'
+#
+# Arguments
+#   file: sysfs file to set (like /sys/class/enclosure/0:0:1:0/SLOT 10/fault)
+#   val: value to set it to
+#
+# Return
+#  0 on success, 3 on missing sysfs path
+#
+check_and_set_led()
+{
+	file="$1"
+	val="$2"
+
+	if [ -z "$val" ]; then
+		return 0
+	fi
+
+	if [ ! -e "$file" ] ; then
+		return 3
+	fi
+
+	# If another process is accessing the LED when we attempt to update it,
+	# the update will be lost so retry until the LED actually changes or we
+	# timeout.
+	for _ in 1 2 3 4 5; do
+		# We want to check the current state first, since writing to the
+		# 'fault' entry always causes a SES command, even if the
+		# current state is already what you want.
+		read -r current < "${file}"
+
+		# On some enclosures if you write 1 to fault, and read it back,
+		# it will return 2.  Treat all non-zero values as 1 for
+		# simplicity.
+		if [ "$current" != "0" ] ; then
+			current=1
+		fi
+
+		if [ "$current" != "$val" ] ; then
+			echo "$val" > "$file"
+			zed_log_msg "vdev $vdev set '$file' LED to $val"
+		else
+			break
+		fi
+	done
+}
+
+# Fault LEDs for JBODs and NVMe drives are handled a little differently.
+#
+# On JBODs the fault LED is called 'fault' and on a path like this:
+#
+#   /sys/class/enclosure/0:0:1:0/SLOT 10/fault
+#
+# On NVMe it's called 'attention' and on a path like this:
+#
+#   /sys/bus/pci/slot/0/attention
+#
+# This function returns the full path to the fault LED file for a given
+# enclosure/slot directory.
+#
+path_to_led()
+{
+	dir=$1
+	if [ -f "$dir/fault" ] ; then
+		echo "$dir/fault"
+	elif [ -f "$dir/attention" ] ; then
+		echo "$dir/attention"
+	fi
+}
+
+state_to_val()
+{
+	state="$1"
+	case "$state" in
+		FAULTED|DEGRADED|UNAVAIL|REMOVED)
+			echo 1
+			;;
+		ONLINE)
+			echo 0
+			;;
+		*)
+			echo "invalid state: $state"
+			;;
+	esac
+}
+
+#
+# Given a nvme name like 'nvme0n1', pass back its slot directory
+# like "/sys/bus/pci/slots/0"
+#
+nvme_dev_to_slot()
+{
+	dev="$1"
+
+	# Get the address "0000:01:00.0"
+	read -r address < "/sys/class/block/$dev/device/address"
+
+	find /sys/bus/pci/slots -regex '.*/[0-9]+/address$' | \
+		while read -r sys_addr; do
+			read -r this_address < "$sys_addr"
+
+			# The format of address is a little different between
+			# /sys/class/block/$dev/device/address and
+			# /sys/bus/pci/slots/
+			#
+			# address=           "0000:01:00.0"
+			# this_address =     "0000:01:00"
+			#
+			if echo "$address" | grep -Eq ^"$this_address" ; then
+				echo "${sys_addr%/*}"
+				break
+			fi
+			done
+}
+
+
+# process_pool (pool)
+#
+# Iterate through a pool and set the vdevs' enclosure slot LEDs to
+# those vdevs' state.
+#
+# Arguments
+#   pool:	Pool name.
+#
+# Return
+#  0 on success, 3 on missing sysfs path
+#
+process_pool()
+{
+	pool="$1"
+
+	# The output will be the vdevs only (from "grep '/dev/'"):
+	#
+	#    U45     ONLINE       0     0     0   /dev/sdk          0
+	#    U46     ONLINE       0     0     0   /dev/sdm          0
+	#    U47     ONLINE       0     0     0   /dev/sdn          0
+	#    U50     ONLINE       0     0     0  /dev/sdbn          0
+	#
+	ZPOOL_SCRIPTS_AS_ROOT=1 $ZPOOL status -c upath,fault_led "$pool" | grep '/dev/' | (
+	rc=0
+	while read -r vdev state _ _ _ therest; do
+		# Read out current LED value and path
+		# Get dev name (like 'sda')
+		dev=$(basename "$(echo "$therest" | awk '{print $(NF-1)}')")
+		vdev_enc_sysfs_path=$(realpath "/sys/class/block/$dev/device/enclosure_device"*)
+		if [ ! -d "$vdev_enc_sysfs_path" ] ; then
+			# This is not a JBOD disk, but it could be a PCI NVMe drive
+			vdev_enc_sysfs_path=$(nvme_dev_to_slot "$dev")
+		fi
+
+		current_val=$(echo "$therest" | awk '{print $NF}')
+
+		if [ "$current_val" != "0" ] ; then
+			current_val=1
+		fi
+
+		if [ -z "$vdev_enc_sysfs_path" ] ; then
+			# Skip anything with no sysfs LED entries
+			continue
+		fi
+
+		led_path=$(path_to_led "$vdev_enc_sysfs_path")
+		if [ ! -e "$led_path" ] ; then
+			rc=3
+			zed_log_msg "vdev $vdev '$led_path' doesn't exist"
+			continue
+		fi
+
+		val=$(state_to_val "$state")
+
+		if [ "$current_val" = "$val" ] ; then
+			# LED is already set correctly
+			continue
+		fi
+
+		if ! check_and_set_led "$led_path" "$val"; then
+			rc=3
+		fi
+	done
+	exit "$rc"; )
+}
+
+if [ -n "$ZEVENT_VDEV_ENC_SYSFS_PATH" ] && [ -n "$ZEVENT_VDEV_STATE_STR" ] ; then
+	# Got a statechange for an individual vdev
+	val=$(state_to_val "$ZEVENT_VDEV_STATE_STR")
+	vdev=$(basename "$ZEVENT_VDEV_PATH")
+	ledpath=$(path_to_led "$ZEVENT_VDEV_ENC_SYSFS_PATH")
+	check_and_set_led "$ledpath" "$val"
+else
+	# Process the entire pool
+	poolname=$(zed_guid_to_pool "$ZEVENT_POOL_GUID")
+	process_pool "$poolname"
+fi
diff --git a/straper/db/public/zfs/etc-zfs/zed.d/statechange-notify.sh b/straper/db/public/zfs/etc-zfs/zed.d/statechange-notify.sh
new file mode 100755
index 0000000..c8c8323
--- /dev/null
+++ b/straper/db/public/zfs/etc-zfs/zed.d/statechange-notify.sh
@@ -0,0 +1,77 @@
+#!/bin/sh
+# SPDX-License-Identifier: CDDL-1.0
+# shellcheck disable=SC2154
+#
+# CDDL HEADER START
+#
+# The contents of this file are subject to the terms of the
+# Common Development and Distribution License Version 1.0 (CDDL-1.0).
+# You can obtain a copy of the license from the top-level file
+# "OPENSOLARIS.LICENSE" or at <http://opensource.org/licenses/CDDL-1.0>.
+# You may not use this file except in compliance with the license.
+#
+# CDDL HEADER END
+#
+
+#
+# Send notification in response to a fault induced statechange
+#
+# ZEVENT_SUBCLASS: 'statechange'
+# ZEVENT_VDEV_STATE_STR: 'DEGRADED', 'FAULTED', 'REMOVED', or 'UNAVAIL'
+#
+# Exit codes:
+#   0: notification sent
+#   1: notification failed
+#   2: notification not configured
+#   3: statechange not relevant
+#   4: statechange string missing (unexpected)
+
+[ -f "${ZED_ZEDLET_DIR}/zed.rc" ] && . "${ZED_ZEDLET_DIR}/zed.rc"
+. "${ZED_ZEDLET_DIR}/zed-functions.sh"
+
+[ -n "${ZEVENT_VDEV_STATE_STR}" ] || exit 4
+
+if [ "${ZEVENT_VDEV_STATE_STR}" != "FAULTED" ] \
+        && [ "${ZEVENT_VDEV_STATE_STR}" != "DEGRADED" ] \
+        && [ "${ZEVENT_VDEV_STATE_STR}" != "REMOVED" ] \
+        && [ "${ZEVENT_VDEV_STATE_STR}" != "UNAVAIL" ]; then
+    exit 3
+fi
+
+umask 077
+note_subject="ZFS device fault for pool ${ZEVENT_POOL} on $(hostname)"
+note_pathname="$(mktemp)"
+{
+    if [ "${ZEVENT_VDEV_STATE_STR}" = "FAULTED" ] ; then
+        echo "The number of I/O errors associated with a ZFS device exceeded"
+        echo "acceptable levels. ZFS has marked the device as faulted."
+    elif [ "${ZEVENT_VDEV_STATE_STR}" = "DEGRADED" ] ; then
+        echo "The number of checksum errors associated with a ZFS device"
+        echo "exceeded acceptable levels. ZFS has marked the device as"
+        echo "degraded."
+    else
+        echo "ZFS has detected that a device was removed."
+    fi
+
+    echo
+    echo " impact: Fault tolerance of the pool may be compromised."
+    echo "    eid: ${ZEVENT_EID}"
+    echo "  class: ${ZEVENT_SUBCLASS}"
+    echo "  state: ${ZEVENT_VDEV_STATE_STR}"
+    echo "   host: $(hostname)"
+    echo "   time: ${ZEVENT_TIME_STRING}"
+
+    [ -n "${ZEVENT_VDEV_TYPE}" ] && echo "  vtype: ${ZEVENT_VDEV_TYPE}"
+    [ -n "${ZEVENT_VDEV_PATH}" ] && echo "  vpath: ${ZEVENT_VDEV_PATH}"
+    [ -n "${ZEVENT_VDEV_PHYSPATH}" ] && echo "  vphys: ${ZEVENT_VDEV_PHYSPATH}"
+    [ -n "${ZEVENT_VDEV_GUID}" ] && echo "  vguid: ${ZEVENT_VDEV_GUID}"
+    [ -n "${ZEVENT_VDEV_DEVID}" ] && echo "  devid: ${ZEVENT_VDEV_DEVID}"
+
+    echo "   pool: ${ZEVENT_POOL} (${ZEVENT_POOL_GUID})"
+
+} > "${note_pathname}"
+
+zed_notify "${note_subject}" "${note_pathname}"; rv=$?
+
+rm -f "${note_pathname}"
+exit "${rv}"
diff --git a/straper/db/public/zfs/etc-zfs/zed.d/statechange-slot_off.sh b/straper/db/public/zfs/etc-zfs/zed.d/statechange-slot_off.sh
new file mode 100755
index 0000000..06acce9
--- /dev/null
+++ b/straper/db/public/zfs/etc-zfs/zed.d/statechange-slot_off.sh
@@ -0,0 +1,64 @@
+#!/bin/sh
+# shellcheck disable=SC3014,SC2154,SC2086,SC2034
+#
+# Turn off disk's enclosure slot if it becomes FAULTED.
+#
+# Bad SCSI disks can often "disappear and reappear" causing all sorts of chaos
+# as they flip between FAULTED and ONLINE.  If
+# ZED_POWER_OFF_ENCLOSURE_SLOT_ON_FAULT is set in zed.rc, and the disk gets
+# FAULTED, then power down the slot via sysfs:
+#
+# /sys/class/enclosure/<enclosure>/<slot>/power_status
+#
+# We assume the user will be responsible for turning the slot back on again.
+#
+# Note that this script requires that your enclosure be supported by the
+# Linux SCSI Enclosure services (SES) driver.  The script will do nothing
+# if you have no enclosure, or if your enclosure isn't supported.
+#
+# Exit codes:
+#   0: slot successfully powered off
+#   1: enclosure not available
+#   2: ZED_POWER_OFF_ENCLOSURE_SLOT_ON_FAULT disabled
+#   3: vdev was not FAULTED
+#   4: The enclosure sysfs path passed from ZFS does not exist
+#   5: Enclosure slot didn't actually turn off after we told it to
+
+[ -f "${ZED_ZEDLET_DIR}/zed.rc" ] && . "${ZED_ZEDLET_DIR}/zed.rc"
+. "${ZED_ZEDLET_DIR}/zed-functions.sh"
+
+if [ ! -d /sys/class/enclosure ] ; then
+	# No JBOD enclosure or NVMe slots
+	exit 1
+fi
+
+if [ "${ZED_POWER_OFF_ENCLOSURE_SLOT_ON_FAULT}" != "1" ] ; then
+	exit 2
+fi
+
+if [ "$ZEVENT_VDEV_STATE_STR" != "FAULTED" ] ; then
+	exit 3
+fi
+
+if [ ! -f "$ZEVENT_VDEV_ENC_SYSFS_PATH/power_status" ] ; then
+	exit 4
+fi
+
+# Turn off the slot and wait for sysfs to report that the slot is off.
+# It can take ~400ms on some enclosures and multiple retries may be needed.
+for i in $(seq 1 20) ; do
+	echo "off" | tee "$ZEVENT_VDEV_ENC_SYSFS_PATH/power_status"
+
+	for j in $(seq 1 5) ; do
+		if [ "$(cat $ZEVENT_VDEV_ENC_SYSFS_PATH/power_status)" == "off" ] ; then
+			break 2
+		fi
+		sleep 0.1
+	done
+done
+
+if [ "$(cat $ZEVENT_VDEV_ENC_SYSFS_PATH/power_status)" != "off" ] ; then
+	exit 5
+fi
+
+zed_log_msg "powered down slot $ZEVENT_VDEV_ENC_SYSFS_PATH for $ZEVENT_VDEV_PATH"
diff --git a/straper/db/public/zfs/etc-zfs/zed.d/vdev_attach-led.sh b/straper/db/public/zfs/etc-zfs/zed.d/vdev_attach-led.sh
new file mode 100755
index 0000000..40cb61f
--- /dev/null
+++ b/straper/db/public/zfs/etc-zfs/zed.d/vdev_attach-led.sh
@@ -0,0 +1,242 @@
+#!/bin/sh
+# shellcheck disable=SC2154
+#
+# Turn off/on vdevs' enclosure fault LEDs when their pool's state changes.
+#
+# Turn a vdev's fault LED on if it becomes FAULTED, DEGRADED or UNAVAIL.
+# Turn its LED off when it's back ONLINE again.
+#
+# This script run in two basic modes:
+#
+# 1. If $ZEVENT_VDEV_ENC_SYSFS_PATH and $ZEVENT_VDEV_STATE_STR are set, then
+# only set the LED for that particular vdev. This is the case for statechange
+# events and some vdev_* events.
+#
+# 2. If those vars are not set, then check the state of all vdevs in the pool
+# and set the LEDs accordingly.  This is the case for pool_import events.
+#
+# Note that this script requires that your enclosure be supported by the
+# Linux SCSI Enclosure services (SES) driver.  The script will do nothing
+# if you have no enclosure, or if your enclosure isn't supported.
+#
+# Exit codes:
+#   0: enclosure led successfully set
+#   1: enclosure leds not available
+#   2: enclosure leds administratively disabled
+#   3: The led sysfs path passed from ZFS does not exist
+#   4: $ZPOOL not set
+#   5: awk is not installed
+
+[ -f "${ZED_ZEDLET_DIR}/zed.rc" ] && . "${ZED_ZEDLET_DIR}/zed.rc"
+. "${ZED_ZEDLET_DIR}/zed-functions.sh"
+
+if [ ! -d /sys/class/enclosure ] && [ ! -d /sys/bus/pci/slots ] ; then
+	# No JBOD enclosure or NVMe slots
+	exit 1
+fi
+
+if [ "${ZED_USE_ENCLOSURE_LEDS}" != "1" ] ; then
+	exit 2
+fi
+
+zed_check_cmd "$ZPOOL" || exit 4
+zed_check_cmd awk || exit 5
+
+# Global used in set_led debug print
+vdev=""
+
+# check_and_set_led (file, val)
+#
+# Read an enclosure sysfs file, and write it if it's not already set to 'val'
+#
+# Arguments
+#   file: sysfs file to set (like /sys/class/enclosure/0:0:1:0/SLOT 10/fault)
+#   val: value to set it to
+#
+# Return
+#  0 on success, 3 on missing sysfs path
+#
+check_and_set_led()
+{
+	file="$1"
+	val="$2"
+
+	if [ -z "$val" ]; then
+		return 0
+	fi
+
+	if [ ! -e "$file" ] ; then
+		return 3
+	fi
+
+	# If another process is accessing the LED when we attempt to update it,
+	# the update will be lost so retry until the LED actually changes or we
+	# timeout.
+	for _ in 1 2 3 4 5; do
+		# We want to check the current state first, since writing to the
+		# 'fault' entry always causes a SES command, even if the
+		# current state is already what you want.
+		read -r current < "${file}"
+
+		# On some enclosures if you write 1 to fault, and read it back,
+		# it will return 2.  Treat all non-zero values as 1 for
+		# simplicity.
+		if [ "$current" != "0" ] ; then
+			current=1
+		fi
+
+		if [ "$current" != "$val" ] ; then
+			echo "$val" > "$file"
+			zed_log_msg "vdev $vdev set '$file' LED to $val"
+		else
+			break
+		fi
+	done
+}
+
+# Fault LEDs for JBODs and NVMe drives are handled a little differently.
+#
+# On JBODs the fault LED is called 'fault' and on a path like this:
+#
+#   /sys/class/enclosure/0:0:1:0/SLOT 10/fault
+#
+# On NVMe it's called 'attention' and on a path like this:
+#
+#   /sys/bus/pci/slot/0/attention
+#
+# This function returns the full path to the fault LED file for a given
+# enclosure/slot directory.
+#
+path_to_led()
+{
+	dir=$1
+	if [ -f "$dir/fault" ] ; then
+		echo "$dir/fault"
+	elif [ -f "$dir/attention" ] ; then
+		echo "$dir/attention"
+	fi
+}
+
+state_to_val()
+{
+	state="$1"
+	case "$state" in
+		FAULTED|DEGRADED|UNAVAIL|REMOVED)
+			echo 1
+			;;
+		ONLINE)
+			echo 0
+			;;
+		*)
+			echo "invalid state: $state"
+			;;
+	esac
+}
+
+#
+# Given a nvme name like 'nvme0n1', pass back its slot directory
+# like "/sys/bus/pci/slots/0"
+#
+nvme_dev_to_slot()
+{
+	dev="$1"
+
+	# Get the address "0000:01:00.0"
+	read -r address < "/sys/class/block/$dev/device/address"
+
+	find /sys/bus/pci/slots -regex '.*/[0-9]+/address$' | \
+		while read -r sys_addr; do
+			read -r this_address < "$sys_addr"
+
+			# The format of address is a little different between
+			# /sys/class/block/$dev/device/address and
+			# /sys/bus/pci/slots/
+			#
+			# address=           "0000:01:00.0"
+			# this_address =     "0000:01:00"
+			#
+			if echo "$address" | grep -Eq ^"$this_address" ; then
+				echo "${sys_addr%/*}"
+				break
+			fi
+			done
+}
+
+
+# process_pool (pool)
+#
+# Iterate through a pool and set the vdevs' enclosure slot LEDs to
+# those vdevs' state.
+#
+# Arguments
+#   pool:	Pool name.
+#
+# Return
+#  0 on success, 3 on missing sysfs path
+#
+process_pool()
+{
+	pool="$1"
+
+	# The output will be the vdevs only (from "grep '/dev/'"):
+	#
+	#    U45     ONLINE       0     0     0   /dev/sdk          0
+	#    U46     ONLINE       0     0     0   /dev/sdm          0
+	#    U47     ONLINE       0     0     0   /dev/sdn          0
+	#    U50     ONLINE       0     0     0  /dev/sdbn          0
+	#
+	ZPOOL_SCRIPTS_AS_ROOT=1 $ZPOOL status -c upath,fault_led "$pool" | grep '/dev/' | (
+	rc=0
+	while read -r vdev state _ _ _ therest; do
+		# Read out current LED value and path
+		# Get dev name (like 'sda')
+		dev=$(basename "$(echo "$therest" | awk '{print $(NF-1)}')")
+		vdev_enc_sysfs_path=$(realpath "/sys/class/block/$dev/device/enclosure_device"*)
+		if [ ! -d "$vdev_enc_sysfs_path" ] ; then
+			# This is not a JBOD disk, but it could be a PCI NVMe drive
+			vdev_enc_sysfs_path=$(nvme_dev_to_slot "$dev")
+		fi
+
+		current_val=$(echo "$therest" | awk '{print $NF}')
+
+		if [ "$current_val" != "0" ] ; then
+			current_val=1
+		fi
+
+		if [ -z "$vdev_enc_sysfs_path" ] ; then
+			# Skip anything with no sysfs LED entries
+			continue
+		fi
+
+		led_path=$(path_to_led "$vdev_enc_sysfs_path")
+		if [ ! -e "$led_path" ] ; then
+			rc=3
+			zed_log_msg "vdev $vdev '$led_path' doesn't exist"
+			continue
+		fi
+
+		val=$(state_to_val "$state")
+
+		if [ "$current_val" = "$val" ] ; then
+			# LED is already set correctly
+			continue
+		fi
+
+		if ! check_and_set_led "$led_path" "$val"; then
+			rc=3
+		fi
+	done
+	exit "$rc"; )
+}
+
+if [ -n "$ZEVENT_VDEV_ENC_SYSFS_PATH" ] && [ -n "$ZEVENT_VDEV_STATE_STR" ] ; then
+	# Got a statechange for an individual vdev
+	val=$(state_to_val "$ZEVENT_VDEV_STATE_STR")
+	vdev=$(basename "$ZEVENT_VDEV_PATH")
+	ledpath=$(path_to_led "$ZEVENT_VDEV_ENC_SYSFS_PATH")
+	check_and_set_led "$ledpath" "$val"
+else
+	# Process the entire pool
+	poolname=$(zed_guid_to_pool "$ZEVENT_POOL_GUID")
+	process_pool "$poolname"
+fi
diff --git a/straper/db/public/zfs/etc-zfs/zed.d/vdev_clear-led.sh b/straper/db/public/zfs/etc-zfs/zed.d/vdev_clear-led.sh
new file mode 100755
index 0000000..40cb61f
--- /dev/null
+++ b/straper/db/public/zfs/etc-zfs/zed.d/vdev_clear-led.sh
@@ -0,0 +1,242 @@
+#!/bin/sh
+# shellcheck disable=SC2154
+#
+# Turn off/on vdevs' enclosure fault LEDs when their pool's state changes.
+#
+# Turn a vdev's fault LED on if it becomes FAULTED, DEGRADED or UNAVAIL.
+# Turn its LED off when it's back ONLINE again.
+#
+# This script run in two basic modes:
+#
+# 1. If $ZEVENT_VDEV_ENC_SYSFS_PATH and $ZEVENT_VDEV_STATE_STR are set, then
+# only set the LED for that particular vdev. This is the case for statechange
+# events and some vdev_* events.
+#
+# 2. If those vars are not set, then check the state of all vdevs in the pool
+# and set the LEDs accordingly.  This is the case for pool_import events.
+#
+# Note that this script requires that your enclosure be supported by the
+# Linux SCSI Enclosure services (SES) driver.  The script will do nothing
+# if you have no enclosure, or if your enclosure isn't supported.
+#
+# Exit codes:
+#   0: enclosure led successfully set
+#   1: enclosure leds not available
+#   2: enclosure leds administratively disabled
+#   3: The led sysfs path passed from ZFS does not exist
+#   4: $ZPOOL not set
+#   5: awk is not installed
+
+[ -f "${ZED_ZEDLET_DIR}/zed.rc" ] && . "${ZED_ZEDLET_DIR}/zed.rc"
+. "${ZED_ZEDLET_DIR}/zed-functions.sh"
+
+if [ ! -d /sys/class/enclosure ] && [ ! -d /sys/bus/pci/slots ] ; then
+	# No JBOD enclosure or NVMe slots
+	exit 1
+fi
+
+if [ "${ZED_USE_ENCLOSURE_LEDS}" != "1" ] ; then
+	exit 2
+fi
+
+zed_check_cmd "$ZPOOL" || exit 4
+zed_check_cmd awk || exit 5
+
+# Global used in set_led debug print
+vdev=""
+
+# check_and_set_led (file, val)
+#
+# Read an enclosure sysfs file, and write it if it's not already set to 'val'
+#
+# Arguments
+#   file: sysfs file to set (like /sys/class/enclosure/0:0:1:0/SLOT 10/fault)
+#   val: value to set it to
+#
+# Return
+#  0 on success, 3 on missing sysfs path
+#
+check_and_set_led()
+{
+	file="$1"
+	val="$2"
+
+	if [ -z "$val" ]; then
+		return 0
+	fi
+
+	if [ ! -e "$file" ] ; then
+		return 3
+	fi
+
+	# If another process is accessing the LED when we attempt to update it,
+	# the update will be lost so retry until the LED actually changes or we
+	# timeout.
+	for _ in 1 2 3 4 5; do
+		# We want to check the current state first, since writing to the
+		# 'fault' entry always causes a SES command, even if the
+		# current state is already what you want.
+		read -r current < "${file}"
+
+		# On some enclosures if you write 1 to fault, and read it back,
+		# it will return 2.  Treat all non-zero values as 1 for
+		# simplicity.
+		if [ "$current" != "0" ] ; then
+			current=1
+		fi
+
+		if [ "$current" != "$val" ] ; then
+			echo "$val" > "$file"
+			zed_log_msg "vdev $vdev set '$file' LED to $val"
+		else
+			break
+		fi
+	done
+}
+
+# Fault LEDs for JBODs and NVMe drives are handled a little differently.
+#
+# On JBODs the fault LED is called 'fault' and on a path like this:
+#
+#   /sys/class/enclosure/0:0:1:0/SLOT 10/fault
+#
+# On NVMe it's called 'attention' and on a path like this:
+#
+#   /sys/bus/pci/slot/0/attention
+#
+# This function returns the full path to the fault LED file for a given
+# enclosure/slot directory.
+#
+path_to_led()
+{
+	dir=$1
+	if [ -f "$dir/fault" ] ; then
+		echo "$dir/fault"
+	elif [ -f "$dir/attention" ] ; then
+		echo "$dir/attention"
+	fi
+}
+
+state_to_val()
+{
+	state="$1"
+	case "$state" in
+		FAULTED|DEGRADED|UNAVAIL|REMOVED)
+			echo 1
+			;;
+		ONLINE)
+			echo 0
+			;;
+		*)
+			echo "invalid state: $state"
+			;;
+	esac
+}
+
+#
+# Given a nvme name like 'nvme0n1', pass back its slot directory
+# like "/sys/bus/pci/slots/0"
+#
+nvme_dev_to_slot()
+{
+	dev="$1"
+
+	# Get the address "0000:01:00.0"
+	read -r address < "/sys/class/block/$dev/device/address"
+
+	find /sys/bus/pci/slots -regex '.*/[0-9]+/address$' | \
+		while read -r sys_addr; do
+			read -r this_address < "$sys_addr"
+
+			# The format of address is a little different between
+			# /sys/class/block/$dev/device/address and
+			# /sys/bus/pci/slots/
+			#
+			# address=           "0000:01:00.0"
+			# this_address =     "0000:01:00"
+			#
+			if echo "$address" | grep -Eq ^"$this_address" ; then
+				echo "${sys_addr%/*}"
+				break
+			fi
+			done
+}
+
+
+# process_pool (pool)
+#
+# Iterate through a pool and set the vdevs' enclosure slot LEDs to
+# those vdevs' state.
+#
+# Arguments
+#   pool:	Pool name.
+#
+# Return
+#  0 on success, 3 on missing sysfs path
+#
+process_pool()
+{
+	pool="$1"
+
+	# The output will be the vdevs only (from "grep '/dev/'"):
+	#
+	#    U45     ONLINE       0     0     0   /dev/sdk          0
+	#    U46     ONLINE       0     0     0   /dev/sdm          0
+	#    U47     ONLINE       0     0     0   /dev/sdn          0
+	#    U50     ONLINE       0     0     0  /dev/sdbn          0
+	#
+	ZPOOL_SCRIPTS_AS_ROOT=1 $ZPOOL status -c upath,fault_led "$pool" | grep '/dev/' | (
+	rc=0
+	while read -r vdev state _ _ _ therest; do
+		# Read out current LED value and path
+		# Get dev name (like 'sda')
+		dev=$(basename "$(echo "$therest" | awk '{print $(NF-1)}')")
+		vdev_enc_sysfs_path=$(realpath "/sys/class/block/$dev/device/enclosure_device"*)
+		if [ ! -d "$vdev_enc_sysfs_path" ] ; then
+			# This is not a JBOD disk, but it could be a PCI NVMe drive
+			vdev_enc_sysfs_path=$(nvme_dev_to_slot "$dev")
+		fi
+
+		current_val=$(echo "$therest" | awk '{print $NF}')
+
+		if [ "$current_val" != "0" ] ; then
+			current_val=1
+		fi
+
+		if [ -z "$vdev_enc_sysfs_path" ] ; then
+			# Skip anything with no sysfs LED entries
+			continue
+		fi
+
+		led_path=$(path_to_led "$vdev_enc_sysfs_path")
+		if [ ! -e "$led_path" ] ; then
+			rc=3
+			zed_log_msg "vdev $vdev '$led_path' doesn't exist"
+			continue
+		fi
+
+		val=$(state_to_val "$state")
+
+		if [ "$current_val" = "$val" ] ; then
+			# LED is already set correctly
+			continue
+		fi
+
+		if ! check_and_set_led "$led_path" "$val"; then
+			rc=3
+		fi
+	done
+	exit "$rc"; )
+}
+
+if [ -n "$ZEVENT_VDEV_ENC_SYSFS_PATH" ] && [ -n "$ZEVENT_VDEV_STATE_STR" ] ; then
+	# Got a statechange for an individual vdev
+	val=$(state_to_val "$ZEVENT_VDEV_STATE_STR")
+	vdev=$(basename "$ZEVENT_VDEV_PATH")
+	ledpath=$(path_to_led "$ZEVENT_VDEV_ENC_SYSFS_PATH")
+	check_and_set_led "$ledpath" "$val"
+else
+	# Process the entire pool
+	poolname=$(zed_guid_to_pool "$ZEVENT_POOL_GUID")
+	process_pool "$poolname"
+fi
diff --git a/straper/db/public/zfs/etc-zfs/zed.d/zed-functions.sh b/straper/db/public/zfs/etc-zfs/zed.d/zed-functions.sh
new file mode 100644
index 0000000..6e00f15
--- /dev/null
+++ b/straper/db/public/zfs/etc-zfs/zed.d/zed-functions.sh
@@ -0,0 +1,824 @@
+#!/bin/sh
+# shellcheck disable=SC2154,SC3043
+# zed-functions.sh
+#
+# ZED helper functions for use in ZEDLETs
+
+
+# Variable Defaults
+#
+: "${ZED_LOCKDIR:="/var/lock"}"
+: "${ZED_NOTIFY_INTERVAL_SECS:=3600}"
+: "${ZED_NOTIFY_VERBOSE:=0}"
+: "${ZED_RUNDIR:="/var/run"}"
+: "${ZED_SYSLOG_PRIORITY:="daemon.notice"}"
+: "${ZED_SYSLOG_TAG:="zed"}"
+
+ZED_FLOCK_FD=8
+
+
+# zed_check_cmd (cmd, ...)
+#
+# For each argument given, search PATH for the executable command [cmd].
+# Log a message if [cmd] is not found.
+#
+# Arguments
+#   cmd: name of executable command for which to search
+#
+# Return
+#   0 if all commands are found in PATH and are executable
+#   n for a count of the command executables that are not found
+#
+zed_check_cmd()
+{
+    local cmd
+    local rv=0
+
+    for cmd; do
+        if ! command -v "${cmd}" >/dev/null 2>&1; then
+            zed_log_err "\"${cmd}\" not installed"
+            rv=$((rv + 1))
+        fi
+    done
+    return "${rv}"
+}
+
+
+# zed_log_msg (msg, ...)
+#
+# Write all argument strings to the system log.
+#
+# Globals
+#   ZED_SYSLOG_PRIORITY
+#   ZED_SYSLOG_TAG
+#
+# Return
+#   nothing
+#
+zed_log_msg()
+{
+    logger -p "${ZED_SYSLOG_PRIORITY}" -t "${ZED_SYSLOG_TAG}" -- "$@"
+}
+
+
+# zed_log_err (msg, ...)
+#
+# Write an error message to the system log.  This message will contain the
+# script name, EID, and all argument strings.
+#
+# Globals
+#   ZED_SYSLOG_PRIORITY
+#   ZED_SYSLOG_TAG
+#   ZEVENT_EID
+#
+# Return
+#   nothing
+#
+zed_log_err()
+{
+    zed_log_msg "error: ${0##*/}:""${ZEVENT_EID:+" eid=${ZEVENT_EID}:"}" "$@"
+}
+
+
+# zed_lock (lockfile, [fd])
+#
+# Obtain an exclusive (write) lock on [lockfile].  If the lock cannot be
+# immediately acquired, wait until it becomes available.
+#
+# Every zed_lock() must be paired with a corresponding zed_unlock().
+#
+# By default, flock-style locks associate the lockfile with file descriptor 8.
+# The bash manpage warns that file descriptors >9 should be used with care as
+# they may conflict with file descriptors used internally by the shell.  File
+# descriptor 9 is reserved for zed_rate_limit().  If concurrent locks are held
+# within the same process, they must use different file descriptors (preferably
+# decrementing from 8); otherwise, obtaining a new lock with a given file
+# descriptor will release the previous lock associated with that descriptor.
+#
+# Arguments
+#   lockfile: pathname of the lock file; the lock will be stored in
+#     ZED_LOCKDIR unless the pathname contains a "/".
+#   fd: integer for the file descriptor used by flock (OPTIONAL unless holding
+#     concurrent locks)
+#
+# Globals
+#   ZED_FLOCK_FD
+#   ZED_LOCKDIR
+#
+# Return
+#   nothing
+#
+zed_lock()
+{
+    local lockfile="$1"
+    local fd="${2:-${ZED_FLOCK_FD}}"
+    local umask_bak
+    local err
+
+    [ -n "${lockfile}" ] || return
+    if ! expr "${lockfile}" : '.*/' >/dev/null 2>&1; then
+        lockfile="${ZED_LOCKDIR}/${lockfile}"
+    fi
+
+    umask_bak="$(umask)"
+    umask 077
+
+    # Obtain a lock on the file bound to the given file descriptor.
+    #
+    eval "exec ${fd}>> '${lockfile}'"
+    if ! err="$(flock --exclusive "${fd}" 2>&1)"; then
+        zed_log_err "failed to lock \"${lockfile}\": ${err}"
+    fi
+
+    umask "${umask_bak}"
+}
+
+
+# zed_unlock (lockfile, [fd])
+#
+# Release the lock on [lockfile].
+#
+# Arguments
+#   lockfile: pathname of the lock file
+#   fd: integer for the file descriptor used by flock (must match the file
+#     descriptor passed to the zed_lock function call)
+#
+# Globals
+#   ZED_FLOCK_FD
+#   ZED_LOCKDIR
+#
+# Return
+#   nothing
+#
+zed_unlock()
+{
+    local lockfile="$1"
+    local fd="${2:-${ZED_FLOCK_FD}}"
+    local err
+
+    [ -n "${lockfile}" ] || return
+    if ! expr "${lockfile}" : '.*/' >/dev/null 2>&1; then
+        lockfile="${ZED_LOCKDIR}/${lockfile}"
+    fi
+
+    # Release the lock and close the file descriptor.
+    if ! err="$(flock --unlock "${fd}" 2>&1)"; then
+        zed_log_err "failed to unlock \"${lockfile}\": ${err}"
+    fi
+    eval "exec ${fd}>&-"
+}
+
+
+# zed_notify (subject, pathname)
+#
+# Send a notification via all available methods.
+#
+# Arguments
+#   subject: notification subject
+#   pathname: pathname containing the notification message (OPTIONAL)
+#
+# Return
+#   0: notification succeeded via at least one method
+#   1: notification failed
+#   2: no notification methods configured
+#
+zed_notify()
+{
+    local subject="$1"
+    local pathname="$2"
+    local num_success=0
+    local num_failure=0
+
+    zed_notify_email "${subject}" "${pathname}"; rv=$?
+    [ "${rv}" -eq 0 ] && num_success=$((num_success + 1))
+    [ "${rv}" -eq 1 ] && num_failure=$((num_failure + 1))
+
+    zed_notify_pushbullet "${subject}" "${pathname}"; rv=$?
+    [ "${rv}" -eq 0 ] && num_success=$((num_success + 1))
+    [ "${rv}" -eq 1 ] && num_failure=$((num_failure + 1))
+
+    zed_notify_slack_webhook "${subject}" "${pathname}"; rv=$?
+    [ "${rv}" -eq 0 ] && num_success=$((num_success + 1))
+    [ "${rv}" -eq 1 ] && num_failure=$((num_failure + 1))
+
+    zed_notify_pushover "${subject}" "${pathname}"; rv=$?
+    [ "${rv}" -eq 0 ] && num_success=$((num_success + 1))
+    [ "${rv}" -eq 1 ] && num_failure=$((num_failure + 1))
+
+    zed_notify_ntfy "${subject}" "${pathname}"; rv=$?
+    [ "${rv}" -eq 0 ] && num_success=$((num_success + 1))
+    [ "${rv}" -eq 1 ] && num_failure=$((num_failure + 1))
+
+    zed_notify_gotify "${subject}" "${pathname}"; rv=$?
+    [ "${rv}" -eq 0 ] && num_success=$((num_success + 1))
+    [ "${rv}" -eq 1 ] && num_failure=$((num_failure + 1))
+
+    [ "${num_success}" -gt 0 ] && return 0
+    [ "${num_failure}" -gt 0 ] && return 1
+    return 2
+}
+
+
+# zed_notify_email (subject, pathname)
+#
+# Send a notification via email to the address specified by ZED_EMAIL_ADDR.
+#
+# Requires the mail executable to be installed in the standard PATH, or
+# ZED_EMAIL_PROG to be defined with the pathname of an executable capable of
+# reading a message body from stdin.
+#
+# Command-line options to the mail executable can be specified in
+# ZED_EMAIL_OPTS.  This undergoes the following keyword substitutions:
+# - @ADDRESS@ is replaced with the space-delimited recipient email address(es)
+# - @SUBJECT@ is replaced with the notification subject
+#   If @SUBJECT@ was omited here, a "Subject: ..." header will be added to notification
+#
+#
+# Arguments
+#   subject: notification subject
+#   pathname: pathname containing the notification message (OPTIONAL)
+#
+# Globals
+#   ZED_EMAIL_PROG
+#   ZED_EMAIL_OPTS
+#   ZED_EMAIL_ADDR
+#
+# Return
+#   0: notification sent
+#   1: notification failed
+#   2: not configured
+#
+zed_notify_email()
+{
+    local subject="${1:-"ZED notification"}"
+    local pathname="${2:-"/dev/null"}"
+
+    : "${ZED_EMAIL_PROG:="mail"}"
+    : "${ZED_EMAIL_OPTS:="-s '@SUBJECT@' @ADDRESS@"}"
+
+    # For backward compatibility with ZED_EMAIL.
+    if [ -n "${ZED_EMAIL}" ] && [ -z "${ZED_EMAIL_ADDR}" ]; then
+        ZED_EMAIL_ADDR="${ZED_EMAIL}"
+    fi
+    [ -n "${ZED_EMAIL_ADDR}" ] || return 2
+
+    zed_check_cmd "${ZED_EMAIL_PROG}" || return 1
+
+    [ -n "${subject}" ] || return 1
+    if [ ! -r "${pathname}" ]; then
+        zed_log_err \
+                "${ZED_EMAIL_PROG##*/} cannot read \"${pathname}\""
+        return 1
+    fi
+
+    # construct cmdline options
+    ZED_EMAIL_OPTS_PARSED="$(echo "${ZED_EMAIL_OPTS}" \
+        | sed   -e "s/@ADDRESS@/${ZED_EMAIL_ADDR}/g" \
+                -e "s/@SUBJECT@/${subject}/g")"
+
+    # pipe message to email prog
+    # shellcheck disable=SC2086,SC2248
+    {
+        # no subject passed as option?
+        if [ "${ZED_EMAIL_OPTS%@SUBJECT@*}" = "${ZED_EMAIL_OPTS}" ] ; then
+            # inject subject header
+            printf "Subject: %s\n" "${subject}"
+            # The following empty line is needed to separate the header from the
+            # body of the message. Otherwise programs like sendmail will skip
+            # everything up to the first empty line (or wont send an email at
+            # all) and will still exit with exit code 0
+            printf "\n"
+        fi
+        # output message
+        cat "${pathname}"
+    } |
+    eval ${ZED_EMAIL_PROG} ${ZED_EMAIL_OPTS_PARSED} >/dev/null 2>&1
+    rv=$?
+    if [ "${rv}" -ne 0 ]; then
+        zed_log_err "${ZED_EMAIL_PROG##*/} exit=${rv}"
+        return 1
+    fi
+    return 0
+}
+
+
+# zed_notify_pushbullet (subject, pathname)
+#
+# Send a notification via Pushbullet <https://www.pushbullet.com/>.
+# The access token (ZED_PUSHBULLET_ACCESS_TOKEN) identifies this client to the
+# Pushbullet server.  The optional channel tag (ZED_PUSHBULLET_CHANNEL_TAG) is
+# for pushing to notification feeds that can be subscribed to; if a channel is
+# not defined, push notifications will instead be sent to all devices
+# associated with the account specified by the access token.
+#
+# Requires awk, curl, and sed executables to be installed in the standard PATH.
+#
+# References
+#   https://docs.pushbullet.com/
+#   https://www.pushbullet.com/security
+#
+# Arguments
+#   subject: notification subject
+#   pathname: pathname containing the notification message (OPTIONAL)
+#
+# Globals
+#   ZED_PUSHBULLET_ACCESS_TOKEN
+#   ZED_PUSHBULLET_CHANNEL_TAG
+#
+# Return
+#   0: notification sent
+#   1: notification failed
+#   2: not configured
+#
+zed_notify_pushbullet()
+{
+    local subject="$1"
+    local pathname="${2:-"/dev/null"}"
+    local msg_body
+    local msg_tag
+    local msg_json
+    local msg_out
+    local msg_err
+    local url="https://api.pushbullet.com/v2/pushes"
+
+    [ -n "${ZED_PUSHBULLET_ACCESS_TOKEN}" ] || return 2
+
+    [ -n "${subject}" ] || return 1
+    if [ ! -r "${pathname}" ]; then
+        zed_log_err "pushbullet cannot read \"${pathname}\""
+        return 1
+    fi
+
+    zed_check_cmd "awk" "curl" "sed" || return 1
+
+    # Escape the following characters in the message body for JSON:
+    # newline, backslash, double quote, horizontal tab, vertical tab,
+    # and carriage return.
+    #
+    msg_body="$(awk '{ ORS="\\n" } { gsub(/\\/, "\\\\"); gsub(/"/, "\\\"");
+        gsub(/\t/, "\\t"); gsub(/\f/, "\\f"); gsub(/\r/, "\\r"); print }' \
+        "${pathname}")"
+
+    # Push to a channel if one is configured.
+    #
+    [ -n "${ZED_PUSHBULLET_CHANNEL_TAG}" ] && msg_tag="$(printf \
+        '"channel_tag": "%s", ' "${ZED_PUSHBULLET_CHANNEL_TAG}")"
+
+    # Construct the JSON message for pushing a note.
+    #
+    msg_json="$(printf '{%s"type": "note", "title": "%s", "body": "%s"}' \
+        "${msg_tag}" "${subject}" "${msg_body}")"
+
+    # Send the POST request and check for errors.
+    #
+    msg_out="$(curl -u "${ZED_PUSHBULLET_ACCESS_TOKEN}:" -X POST "${url}" \
+        --header "Content-Type: application/json" --data-binary "${msg_json}" \
+        2>/dev/null)"; rv=$?
+    if [ "${rv}" -ne 0 ]; then
+        zed_log_err "curl exit=${rv}"
+        return 1
+    fi
+    msg_err="$(echo "${msg_out}" \
+        | sed -n -e 's/.*"error" *:.*"message" *: *"\([^"]*\)".*/\1/p')"
+    if [ -n "${msg_err}" ]; then
+        zed_log_err "pushbullet \"${msg_err}"\"
+        return 1
+    fi
+    return 0
+}
+
+
+# zed_notify_slack_webhook (subject, pathname)
+#
+# Notification via Slack Webhook <https://api.slack.com/incoming-webhooks>.
+# The Webhook URL (ZED_SLACK_WEBHOOK_URL) identifies this client to the
+# Slack channel.
+#
+# Requires awk, curl, and sed executables to be installed in the standard PATH.
+#
+# References
+#   https://api.slack.com/incoming-webhooks
+#
+# Arguments
+#   subject: notification subject
+#   pathname: pathname containing the notification message (OPTIONAL)
+#
+# Globals
+#   ZED_SLACK_WEBHOOK_URL
+#
+# Return
+#   0: notification sent
+#   1: notification failed
+#   2: not configured
+#
+zed_notify_slack_webhook()
+{
+    [ -n "${ZED_SLACK_WEBHOOK_URL}" ] || return 2
+
+    local subject="$1"
+    local pathname="${2:-"/dev/null"}"
+    local msg_body
+    local msg_tag
+    local msg_json
+    local msg_out
+    local msg_err
+    local url="${ZED_SLACK_WEBHOOK_URL}"
+
+    [ -n "${subject}" ] || return 1
+    if [ ! -r "${pathname}" ]; then
+        zed_log_err "slack webhook cannot read \"${pathname}\""
+        return 1
+    fi
+
+    zed_check_cmd "awk" "curl" "sed" || return 1
+
+    # Escape the following characters in the message body for JSON:
+    # newline, backslash, double quote, horizontal tab, vertical tab,
+    # and carriage return.
+    #
+    msg_body="$(awk '{ ORS="\\n" } { gsub(/\\/, "\\\\"); gsub(/"/, "\\\"");
+        gsub(/\t/, "\\t"); gsub(/\f/, "\\f"); gsub(/\r/, "\\r"); print }' \
+        "${pathname}")"
+
+    # Construct the JSON message for posting.
+    #
+    msg_json="$(printf '{"text": "*%s*\\n%s"}' "${subject}" "${msg_body}" )"
+
+    # Send the POST request and check for errors.
+    #
+    msg_out="$(curl -X POST "${url}" \
+        --header "Content-Type: application/json" --data-binary "${msg_json}" \
+        2>/dev/null)"; rv=$?
+    if [ "${rv}" -ne 0 ]; then
+        zed_log_err "curl exit=${rv}"
+        return 1
+    fi
+    msg_err="$(echo "${msg_out}" \
+        | sed -n -e 's/.*"error" *:.*"message" *: *"\([^"]*\)".*/\1/p')"
+    if [ -n "${msg_err}" ]; then
+        zed_log_err "slack webhook \"${msg_err}"\"
+        return 1
+    fi
+    return 0
+}
+
+# zed_notify_pushover (subject, pathname)
+#
+# Send a notification via Pushover <https://pushover.net/>.
+# The access token (ZED_PUSHOVER_TOKEN) identifies this client to the
+# Pushover server. The user token (ZED_PUSHOVER_USER) defines the user or
+# group to which the notification will be sent.
+#
+# Requires curl and sed executables to be installed in the standard PATH.
+#
+# References
+#   https://pushover.net/api
+#
+# Arguments
+#   subject: notification subject
+#   pathname: pathname containing the notification message (OPTIONAL)
+#
+# Globals
+#   ZED_PUSHOVER_TOKEN
+#   ZED_PUSHOVER_USER
+#
+# Return
+#   0: notification sent
+#   1: notification failed
+#   2: not configured
+#
+zed_notify_pushover()
+{
+    local subject="$1"
+    local pathname="${2:-"/dev/null"}"
+    local msg_body
+    local msg_out
+    local msg_err
+    local url="https://api.pushover.net/1/messages.json"
+
+    [ -n "${ZED_PUSHOVER_TOKEN}" ] && [ -n "${ZED_PUSHOVER_USER}" ] || return 2
+
+    if [ ! -r "${pathname}" ]; then
+        zed_log_err "pushover cannot read \"${pathname}\""
+        return 1
+    fi
+
+    zed_check_cmd "curl" "sed" || return 1
+
+    # Read the message body in.
+    #
+    msg_body="$(cat "${pathname}")"
+
+    if [ -z "${msg_body}" ]
+    then
+        msg_body=$subject
+        subject=""
+    fi
+
+    # Send the POST request and check for errors.
+    #
+    msg_out="$( \
+        curl \
+        --form-string "token=${ZED_PUSHOVER_TOKEN}" \
+        --form-string "user=${ZED_PUSHOVER_USER}" \
+        --form-string "message=${msg_body}" \
+        --form-string "title=${subject}" \
+        "${url}" \
+        2>/dev/null \
+        )"; rv=$?
+    if [ "${rv}" -ne 0 ]; then
+        zed_log_err "curl exit=${rv}"
+        return 1
+    fi
+    msg_err="$(echo "${msg_out}" \
+        | sed -n -e 's/.*"errors" *:.*\[\(.*\)\].*/\1/p')"
+    if [ -n "${msg_err}" ]; then
+        zed_log_err "pushover \"${msg_err}"\"
+        return 1
+    fi
+    return 0
+}
+
+
+# zed_notify_ntfy (subject, pathname)
+#
+# Send a notification via Ntfy.sh <https://ntfy.sh/>.
+# The ntfy topic (ZED_NTFY_TOPIC) identifies the topic that the notification
+# will be sent to Ntfy.sh server. The ntfy url (ZED_NTFY_URL) defines the
+# self-hosted or provided hosted ntfy service location. The ntfy access token
+# <https://docs.ntfy.sh/publish/#access-tokens> (ZED_NTFY_ACCESS_TOKEN) reprsents an
+# access token that could be used if a topic is read/write protected. If a
+# topic can be written to publicaly, a ZED_NTFY_ACCESS_TOKEN is not required.
+#
+# Requires curl and sed executables to be installed in the standard PATH.
+#
+# References
+#   https://docs.ntfy.sh
+#
+# Arguments
+#   subject: notification subject
+#   pathname: pathname containing the notification message (OPTIONAL)
+#
+# Globals
+#   ZED_NTFY_TOPIC
+#   ZED_NTFY_ACCESS_TOKEN (OPTIONAL)
+#   ZED_NTFY_URL
+#
+# Return
+#   0: notification sent
+#   1: notification failed
+#   2: not configured
+#
+zed_notify_ntfy()
+{
+    local subject="$1"
+    local pathname="${2:-"/dev/null"}"
+    local msg_body
+    local msg_out
+    local msg_err
+
+    [ -n "${ZED_NTFY_TOPIC}" ] || return 2
+    local url="${ZED_NTFY_URL:-"https://ntfy.sh"}/${ZED_NTFY_TOPIC}"
+
+    if [ ! -r "${pathname}" ]; then
+        zed_log_err "ntfy cannot read \"${pathname}\""
+        return 1
+    fi
+
+    zed_check_cmd "curl" "sed" || return 1
+
+    # Read the message body in.
+    #
+    msg_body="$(cat "${pathname}")"
+
+    if [ -z "${msg_body}" ]
+    then
+        msg_body=$subject
+        subject=""
+    fi
+
+    # Send the POST request and check for errors.
+    #
+    if [ -n "${ZED_NTFY_ACCESS_TOKEN}" ]; then
+        msg_out="$( \
+        curl \
+        -u ":${ZED_NTFY_ACCESS_TOKEN}" \
+        -H "Title: ${subject}" \
+        -d "${msg_body}" \
+        -H "Priority: high" \
+        "${url}" \
+        2>/dev/null \
+        )"; rv=$?
+    else
+        msg_out="$( \
+        curl \
+        -H "Title: ${subject}" \
+        -d "${msg_body}" \
+        -H "Priority: high" \
+        "${url}" \
+        2>/dev/null \
+        )"; rv=$?
+    fi
+    if [ "${rv}" -ne 0 ]; then
+        zed_log_err "curl exit=${rv}"
+        return 1
+    fi
+    msg_err="$(echo "${msg_out}" \
+        | sed -n -e 's/.*"errors" *:.*\[\(.*\)\].*/\1/p')"
+    if [ -n "${msg_err}" ]; then
+        zed_log_err "ntfy \"${msg_err}"\"
+        return 1
+    fi
+    return 0
+}
+
+
+# zed_notify_gotify (subject, pathname)
+#
+# Send a notification via Gotify <https://gotify.net/>.
+# The Gotify URL (ZED_GOTIFY_URL) defines a self-hosted Gotify location.
+# The Gotify application token (ZED_GOTIFY_APPTOKEN) defines a
+# Gotify application token which is associated with a message.
+# The optional Gotify priority value (ZED_GOTIFY_PRIORITY) overrides the
+# default or configured priority at the Gotify server for the application.
+#
+# Requires curl and sed executables to be installed in the standard PATH.
+#
+# References
+#   https://gotify.net/docs/index
+#
+# Arguments
+#   subject: notification subject
+#   pathname: pathname containing the notification message (OPTIONAL)
+#
+# Globals
+#   ZED_GOTIFY_URL
+#   ZED_GOTIFY_APPTOKEN
+#   ZED_GOTIFY_PRIORITY
+#
+# Return
+#   0: notification sent
+#   1: notification failed
+#   2: not configured
+#
+zed_notify_gotify()
+{
+    local subject="$1"
+    local pathname="${2:-"/dev/null"}"
+    local msg_body
+    local msg_out
+    local msg_err
+
+    [ -n "${ZED_GOTIFY_URL}" ] && [ -n "${ZED_GOTIFY_APPTOKEN}" ] || return 2
+    local url="${ZED_GOTIFY_URL}/message?token=${ZED_GOTIFY_APPTOKEN}"
+
+    if [ ! -r "${pathname}" ]; then
+        zed_log_err "gotify cannot read \"${pathname}\""
+        return 1
+    fi
+
+    zed_check_cmd "curl" "sed" || return 1
+
+    # Read the message body in.
+    #
+    msg_body="$(cat "${pathname}")"
+
+    if [ -z "${msg_body}" ]
+    then
+        msg_body=$subject
+        subject=""
+    fi
+
+    # Send the POST request and check for errors.
+    #
+    if [ -n "${ZED_GOTIFY_PRIORITY}" ]; then
+        msg_out="$( \
+        curl \
+        --form-string "title=${subject}" \
+        --form-string "message=${msg_body}" \
+        --form-string "priority=${ZED_GOTIFY_PRIORITY}" \
+        "${url}" \
+        2>/dev/null \
+        )"; rv=$?
+    else
+        msg_out="$( \
+        curl \
+        --form-string "title=${subject}" \
+        --form-string "message=${msg_body}" \
+        "${url}" \
+        2>/dev/null \
+        )"; rv=$?
+    fi
+
+    if [ "${rv}" -ne 0 ]; then
+        zed_log_err "curl exit=${rv}"
+        return 1
+    fi
+    msg_err="$(echo "${msg_out}" \
+        | sed -n -e 's/.*"errors" *:.*\[\(.*\)\].*/\1/p')"
+    if [ -n "${msg_err}" ]; then
+        zed_log_err "gotify \"${msg_err}"\"
+        return 1
+    fi
+    return 0
+}
+
+
+
+# zed_rate_limit (tag, [interval])
+#
+# Check whether an event of a given type [tag] has already occurred within the
+# last [interval] seconds.
+#
+# This function obtains a lock on the statefile using file descriptor 9.
+#
+# Arguments
+#   tag: arbitrary string for grouping related events to rate-limit
+#   interval: time interval in seconds (OPTIONAL)
+#
+# Globals
+#   ZED_NOTIFY_INTERVAL_SECS
+#   ZED_RUNDIR
+#
+# Return
+#   0 if the event should be processed
+#   1 if the event should be dropped
+#
+# State File Format
+#   time;tag
+#
+zed_rate_limit()
+{
+    local tag="$1"
+    local interval="${2:-${ZED_NOTIFY_INTERVAL_SECS}}"
+    local lockfile="zed.zedlet.state.lock"
+    local lockfile_fd=9
+    local statefile="${ZED_RUNDIR}/zed.zedlet.state"
+    local time_now
+    local time_prev
+    local umask_bak
+    local rv=0
+
+    [ -n "${tag}" ] || return 0
+
+    zed_lock "${lockfile}" "${lockfile_fd}"
+    time_now="$(date +%s)"
+    time_prev="$(grep -E "^[0-9]+;${tag}\$" "${statefile}" 2>/dev/null \
+        | tail -1 | cut -d\; -f1)"
+
+    if [ -n "${time_prev}" ] \
+            && [ "$((time_now - time_prev))" -lt "${interval}" ]; then
+        rv=1
+    else
+        umask_bak="$(umask)"
+        umask 077
+        grep -E -v "^[0-9]+;${tag}\$" "${statefile}" 2>/dev/null \
+            > "${statefile}.$$"
+        echo "${time_now};${tag}" >> "${statefile}.$$"
+        mv -f "${statefile}.$$" "${statefile}"
+        umask "${umask_bak}"
+    fi
+
+    zed_unlock "${lockfile}" "${lockfile_fd}"
+    return "${rv}"
+}
+
+
+# zed_guid_to_pool (guid)
+#
+# Convert a pool GUID into its pool name (like "tank")
+# Arguments
+#   guid: pool GUID (decimal or hex)
+#
+# Return
+#   Pool name
+#
+zed_guid_to_pool()
+{
+	if [ -z "$1" ] ; then
+		return
+	fi
+
+	guid="$(printf "%u" "$1")"
+	$ZPOOL get -H -ovalue,name guid | awk '$1 == '"$guid"' {print $2; exit}'
+}
+
+# zed_exit_if_ignoring_this_event
+#
+# Exit the script if we should ignore this event, as determined by
+# $ZED_SYSLOG_SUBCLASS_INCLUDE and $ZED_SYSLOG_SUBCLASS_EXCLUDE in zed.rc.
+# This function assumes you've imported the normal zed variables.
+zed_exit_if_ignoring_this_event()
+{
+	if [ -n "${ZED_SYSLOG_SUBCLASS_INCLUDE}" ]; then
+	    eval "case ${ZEVENT_SUBCLASS} in
+	    ${ZED_SYSLOG_SUBCLASS_INCLUDE});;
+	    *) exit 0;;
+	    esac"
+	elif [ -n "${ZED_SYSLOG_SUBCLASS_EXCLUDE}" ]; then
+	    eval "case ${ZEVENT_SUBCLASS} in
+	    ${ZED_SYSLOG_SUBCLASS_EXCLUDE}) exit 0;;
+	    *);;
+	    esac"
+	fi
+}
diff --git a/straper/db/public/zfs/etc-zfs/zed.d/zed.rc b/straper/db/public/zfs/etc-zfs/zed.d/zed.rc
new file mode 100644
index 0000000..47fb016
--- /dev/null
+++ b/straper/db/public/zfs/etc-zfs/zed.d/zed.rc
@@ -0,0 +1,199 @@
+##
+# zed.rc – ZEDLET configuration.
+##
+# shellcheck disable=SC2034
+
+##
+# Absolute path to the debug output file.
+#
+#ZED_DEBUG_LOG="/tmp/zed.debug.log"
+
+##
+# Email address of the zpool administrator for receipt of notifications;
+#   multiple addresses can be specified if they are delimited by whitespace.
+# Email will only be sent if ZED_EMAIL_ADDR is defined.
+# Enabled by default; comment to disable.
+#
+ZED_EMAIL_ADDR="root"
+
+##
+# Name or path of executable responsible for sending notifications via email;
+#   the mail program must be capable of reading a message body from stdin.
+# Email will only be sent if ZED_EMAIL_ADDR is defined.
+#
+#ZED_EMAIL_PROG="mail"
+
+##
+# Command-line options for ZED_EMAIL_PROG.
+# The string @ADDRESS@ will be replaced with the recipient email address(es).
+# The string @SUBJECT@ will be replaced with the notification subject;
+#   this should be protected with quotes to prevent word-splitting.
+# Email will only be sent if ZED_EMAIL_ADDR is defined.
+# If @SUBJECT@ was omited here, a "Subject: ..." header will be added to notification
+#
+#ZED_EMAIL_OPTS="-s '@SUBJECT@' @ADDRESS@"
+
+##
+# Default directory for zed lock files.
+#
+#ZED_LOCKDIR="/var/lock"
+
+##
+# Minimum number of seconds between notifications for a similar event.
+#
+ZED_NOTIFY_INTERVAL_SECS=3600
+
+##
+# Notification verbosity.
+#   If set to 0, suppress notification if the pool is healthy.
+#   If set to 1, send notification regardless of pool health.
+#
+#ZED_NOTIFY_VERBOSE=0
+
+##
+# Send notifications for 'ereport.fs.zfs.data' events.
+# Disabled by default, any non-empty value will enable the feature.
+#
+#ZED_NOTIFY_DATA=
+
+##
+# Pushbullet access token.
+# This grants full access to your account -- protect it accordingly!
+#   <https://www.pushbullet.com/get-started>
+#   <https://www.pushbullet.com/account>
+# Disabled by default; uncomment to enable.
+#
+#ZED_PUSHBULLET_ACCESS_TOKEN=""
+
+##
+# Pushbullet channel tag for push notification feeds that can be subscribed to.
+#   <https://www.pushbullet.com/my-channel>
+# If not defined, push notifications will instead be sent to all devices
+#   associated with the account specified by the access token.
+# Disabled by default; uncomment to enable.
+#
+#ZED_PUSHBULLET_CHANNEL_TAG=""
+
+##
+# Slack Webhook URL.
+# This allows posting to the given channel and includes an access token.
+#   <https://api.slack.com/incoming-webhooks>
+# Disabled by default; uncomment to enable.
+#
+#ZED_SLACK_WEBHOOK_URL=""
+
+##
+# Pushover token.
+# This defines the application from which the notification will be sent.
+#   <https://pushover.net/api#registration>
+# Disabled by default; uncomment to enable.
+# ZED_PUSHOVER_USER, below, must also be configured.
+#
+#ZED_PUSHOVER_TOKEN=""
+
+##
+# Pushover user key.
+# This defines which user or group will receive Pushover notifications.
+#  <https://pushover.net/api#identifiers>
+# Disabled by default; uncomment to enable.
+# ZED_PUSHOVER_TOKEN, above, must also be configured.
+#ZED_PUSHOVER_USER=""
+
+##
+# Default directory for zed state files.
+#
+#ZED_RUNDIR="/var/run"
+
+##
+# Turn on/off enclosure LEDs when drives get DEGRADED/FAULTED.  This works for
+# device mapper and multipath devices as well.  This works with JBOD enclosures
+# and NVMe PCI drives (assuming they're supported by Linux in sysfs).
+#
+ZED_USE_ENCLOSURE_LEDS=1
+
+##
+# Run a scrub after every resilver
+# Disabled by default, 1 to enable and 0 to disable.
+#ZED_SCRUB_AFTER_RESILVER=0
+
+##
+# The syslog priority (e.g., specified as a "facility.level" pair).
+#
+#ZED_SYSLOG_PRIORITY="daemon.notice"
+
+##
+# The syslog tag for marking zed events.
+#
+#ZED_SYSLOG_TAG="zed"
+
+##
+# Which set of event subclasses to log
+# By default, events from all subclasses are logged.
+# If ZED_SYSLOG_SUBCLASS_INCLUDE is set, only subclasses
+# matching the pattern are logged. Use the pipe symbol (|)
+# or shell wildcards (*, ?) to match multiple subclasses.
+# Otherwise, if ZED_SYSLOG_SUBCLASS_EXCLUDE is set, the
+# matching subclasses are excluded from logging.
+#ZED_SYSLOG_SUBCLASS_INCLUDE="checksum|scrub_*|vdev.*"
+ZED_SYSLOG_SUBCLASS_EXCLUDE="history_event"
+
+##
+# Use GUIDs instead of names when logging pool and vdevs
+# Disabled by default, 1 to enable and 0 to disable.
+#ZED_SYSLOG_DISPLAY_GUIDS=1
+
+##
+# Power off the drive's slot in the enclosure if it becomes FAULTED.  This can
+# help silence misbehaving drives.  This assumes your drive enclosure fully
+# supports slot power control via sysfs.
+#ZED_POWER_OFF_ENCLOSURE_SLOT_ON_FAULT=1
+
+##
+# Power off the drive's slot in the enclosure if there is a hung I/O which
+# exceeds the deadman timeout.  This can help prevent a single misbehaving
+# drive from rendering a redundant pool unavailable.  This assumes your drive
+# enclosure fully supports slot power control via sysfs.
+#ZED_POWER_OFF_ENCLOSURE_SLOT_ON_DEADMAN=1
+
+##
+# Ntfy topic
+# This defines which topic will receive the ntfy notification.
+#  <https://docs.ntfy.sh/publish/>
+# Disabled by default; uncomment to enable.
+#ZED_NTFY_TOPIC=""
+
+##
+# Ntfy access token (optional for public topics)
+# This defines an access token which can be used
+# to allow you to authenticate when sending to topics
+# <https://docs.ntfy.sh/publish/#access-tokens>
+# Disabled by default; uncomment to enable.
+#ZED_NTFY_ACCESS_TOKEN=""
+
+##
+# Ntfy Service URL
+# This defines which service the ntfy call will be directed toward
+#  <https://docs.ntfy.sh/install/>
+# https://ntfy.sh by default; uncomment to enable an alternative service url.
+#ZED_NTFY_URL="https://ntfy.sh"
+
+##
+# Gotify server URL
+# This defines a URL that the Gotify call will be directed toward.
+# <https://gotify.net/docs/index>
+# Disabled by default; uncomment to enable.
+#ZED_GOTIFY_URL=""
+
+##
+# Gotify application token
+# This defines a Gotify application token which a message is associated with.
+# This token is generated when an application is created on the Gotify server.
+# Disabled by default; uncomment to enable.
+#ZED_GOTIFY_APPTOKEN=""
+
+##
+# Gotify priority (optional)
+# If defined, this overrides the default priority of the
+# Gotify application associated with ZED_GOTIFY_APPTOKEN.
+# Value is an integer 0 and up.
+#ZED_GOTIFY_PRIORITY=""