KVM: make storage heartbeat fence action configurable (graceful-reboot / restart-agent / log-only)

agent/conf/agent.properties

@@ -318,6 +318,80 @@ iscsi.session.cleanup.enabled=false
 # This parameter specifies if the host must be rebooted when something goes wrong with the heartbeat.
 #reboot.host.and.alert.management.on.heartbeat.timeout=false
 
+# Action taken by kvmheartbeat.sh / kvmspheartbeat.sh / kvmsmpheartbeat.sh when a storage heartbeat
+# write fails persistently. Only consulted when fencing is enabled, i.e.
+# 'reboot.host.and.alert.management.on.heartbeat.timeout' is true: that boolean remains
+# the global on/off switch for fencing, this setting only chooses WHAT the fence does.
+#
+# Allowed values:
+#   hard-reboot     - immediate sysrq-trigger reboot (default).
+#                     Required default for setups where a stale NFSv3 mount can prevent
+#                     a graceful shutdown from completing.
+#   graceful-reboot - 'systemctl reboot' instead of sysrq; allows VMs to stop cleanly.
+#                     Use only if a stale storage mount cannot block shutdown.
+#   restart-agent   - stop every VM on the host, then restart cloudstack-agent; the
+#                     host itself stays up. If it already fired within
+#                     kvm.heartbeat.fence.restart.agent.cooldown, hard-reboot instead.
+#   log-only        - stop every VM on the host, then only write a message to the
+#                     host's syslog (no reboot, no agent restart, no management server
+#                     alert). The operator must watch syslog and investigate.
+#
+# Any other value falls back to hard-reboot. restart-agent and graceful-reboot need
+# systemctl; if it is missing or the systemctl call fails, the fence hard-reboots.
+#
+# restart-agent and log-only stop the VMs because, once the heartbeat is stale, the
+# management server may declare the host Down and HA may start the same VMs on another
+# host; two copies would then write to the same disks. If any VM cannot be stopped,
+# the fence falls back to hard-reboot.
+#
+# The non-default values are recommended for setups using LINSTOR/DRBD or any local
+# storage with replication, where transient I/O contention can cause a heartbeat
+# write to time out without the host actually being unhealthy.
+#kvm.heartbeat.fence.action=hard-reboot
+
+# Set to true to let restart-agent and log-only leave the VMs running. This is only
+# safe when no VM on this host can be started on another host by HA (VM HA off for all
+# of them and host HA not enabled). Otherwise a host the management server declares
+# Down can end up with the same VM running twice and corrupting its disks.
+# Has no effect on hard-reboot and graceful-reboot.
+# This is the master opt-out: when true, nothing is stopped and
+# kvm.heartbeat.fence.include.system.vms is ignored (the syslog warning says so).
+#kvm.heartbeat.fence.keep.vms.running=false
+
+# Whether restart-agent and log-only also stop system VMs (virtual routers, console
+# proxy, secondary storage and internal LB VMs, recognised by their r-, v-, s- and b-
+# instance names). With true (default) every VM on the host is stopped. With false
+# only user VMs are stopped; system VMs keep running and a system VM that is still
+# running does not trigger the hard-reboot fallback.
+# Risk of false: if the management server declares this host Down it restarts the
+# system VMs on another host, so a virtual router could run twice with the same IP
+# and MAC addresses on the guest network, and a second console proxy or secondary
+# storage VM could run with the same addresses. Only set false if you accept that.
+# Ignored when kvm.heartbeat.fence.keep.vms.running is true. Has no effect on
+# hard-reboot and graceful-reboot.
+#kvm.heartbeat.fence.include.system.vms=true
+
+# Seconds restart-agent and log-only may spend stopping VMs before the host is
+# hard-rebooted instead. Before stopping any VM the fence arms a detached deadman that
+# keeps running even if the agent kills the heartbeat script when
+# heartbeat.update.timeout runs out (stopping a VM can block for a long time on hung
+# storage). If the VMs are not all confirmed stopped within this budget, the deadman
+# hard-reboots the host. Each virsh call is also limited to 10 seconds (or this value
+# if lower) when coreutils timeout is installed. Keep it below heartbeat.update.timeout
+# (60 seconds by default). A value that is not a positive integer means 30.
+# Has no effect on hard-reboot and graceful-reboot, or when
+# kvm.heartbeat.fence.keep.vms.running is true.
+#kvm.heartbeat.fence.stop.vms.timeout=30
+
+# Loop guard for restart-agent, in seconds. While storage keeps failing, the restarted
+# agent runs the heartbeat again, it fails again and restart-agent would restart the
+# agent again, over and over. Each restart-agent fence records its time under /run
+# (kept across agent restarts, cleared by a reboot); if restart-agent already fired less
+# than this many seconds ago, the fence hard-reboots the host instead. 0 turns the guard
+# off. A value that is not a whole number of seconds means 600.
+# Has no effect on hard-reboot, graceful-reboot and log-only.
+#kvm.heartbeat.fence.restart.agent.cooldown=600
+
 # Enables manually setting CPU's topology on KVM's VM.
 #enable.manually.setting.cpu.topology.on.kvm.vm=true
 

agent/src/main/java/com/cloud/agent/properties/AgentProperties.java

@@ -622,6 +622,96 @@ public class AgentProperties{
     public static final Property<Boolean> REBOOT_HOST_AND_ALERT_MANAGEMENT_ON_HEARTBEAT_TIMEOUT
         = new Property<>("reboot.host.and.alert.management.on.heartbeat.timeout", false);
 
+    /**
+     * Action taken by the KVM agent's storage heartbeat scripts (kvmheartbeat.sh, kvmspheartbeat.sh
+     * and kvmsmpheartbeat.sh) when a heartbeat write fails persistently. Only consulted when
+     * {@link #REBOOT_HOST_AND_ALERT_MANAGEMENT_ON_HEARTBEAT_TIMEOUT} is true: that boolean remains the
+     * on/off switch for fencing, this setting only chooses what the fence does. Allowed values:
+     * <ul>
+     *   <li>{@code hard-reboot} (default): immediate sysrq-trigger reboot. Required default for setups
+     *       where a stale NFSv3 mount can prevent a graceful shutdown from completing.</li>
+     *   <li>{@code graceful-reboot}: {@code systemctl reboot} instead of sysrq; allows VMs
+     *       to stop cleanly. Use only if a stale storage mount cannot block shutdown.</li>
+     *   <li>{@code restart-agent}: stop every VM on the host, then restart cloudstack-agent; the host stays up. If it
+     *       already fired within {@link #KVM_HEARTBEAT_FENCE_RESTART_AGENT_COOLDOWN}, hard-reboot instead.</li>
+     *   <li>{@code log-only}: stop every VM on the host, then only write a message to the host's syslog (no
+     *       reboot, no agent restart, no management server alert); the operator must investigate.</li>
+     * </ul>
+     * Any other value falls back to {@code hard-reboot}. {@code restart-agent} and {@code graceful-reboot} need
+     * systemctl; if it is missing or the systemctl call fails, the fence hard-reboots.<br>
+     * {@code restart-agent} and {@code log-only} stop the VMs because, once the heartbeat is stale, the management
+     * server may declare the host Down and HA may start the same VMs on another host, where both copies would write
+     * to the same disks. If any VM cannot be stopped the fence falls back to {@code hard-reboot}. See
+     * {@link #KVM_HEARTBEAT_FENCE_KEEP_VMS_RUNNING}.<br>
+     * The non-default values are recommended for setups using LINSTOR/DRBD or other replicated
+     * local storage, where transient I/O contention can cause a heartbeat write to time out
+     * without the host actually being unhealthy.<br>
+     * Read by the heartbeat shell scripts directly from agent.properties.<br>
+     * Data type: String.<br>
+     * Default value: {@code hard-reboot}
+     */
+    public static final Property<String> KVM_HEARTBEAT_FENCE_ACTION
+        = new Property<>("kvm.heartbeat.fence.action", "hard-reboot");
+
+    /**
+     * When true, the {@code restart-agent} and {@code log-only} fence actions leave the host's VMs running instead of
+     * stopping them first. Only safe when no VM on the host can be started on another host by HA (VM HA off for all
+     * of them and host HA not enabled); otherwise a host the management server declares Down can end up with the
+     * same VM running twice, writing to the same disks. No effect on {@code hard-reboot} and {@code graceful-reboot}.
+     * This is the master opt-out: when true nothing is stopped and {@link #KVM_HEARTBEAT_FENCE_INCLUDE_SYSTEM_VMS}
+     * is ignored.<br>
+     * Read by the heartbeat shell scripts directly from agent.properties.<br>
+     * Data type: Boolean.<br>
+     * Default value: <code>false</code>
+     */
+    public static final Property<Boolean> KVM_HEARTBEAT_FENCE_KEEP_VMS_RUNNING
+        = new Property<>("kvm.heartbeat.fence.keep.vms.running", false);
+
+    /**
+     * Whether the {@code restart-agent} and {@code log-only} fence actions also stop system VMs (virtual routers,
+     * console proxy, secondary storage and internal LB VMs, recognised by their {@code r-}, {@code v-}, {@code s-} and
+     * {@code b-} instance names). When true every VM on the host is stopped. When false only user VMs are stopped;
+     * system VMs keep running and do not trigger the {@code hard-reboot} fallback. Risk of false: if the management
+     * server declares the host Down it restarts the system VMs on another host, so a virtual router could run twice
+     * with the same IP and MAC addresses. Ignored when {@link #KVM_HEARTBEAT_FENCE_KEEP_VMS_RUNNING} is true. No
+     * effect on {@code hard-reboot} and {@code graceful-reboot}.
+     * Read by the heartbeat shell scripts directly from agent.properties.<br>
+     * Data type: Boolean.<br>
+     * Default value: <code>true</code>
+     */
+    public static final Property<Boolean> KVM_HEARTBEAT_FENCE_INCLUDE_SYSTEM_VMS
+        = new Property<>("kvm.heartbeat.fence.include.system.vms", true);
+
+    /**
+     * Seconds the {@code restart-agent} and {@code log-only} fence actions may spend stopping VMs before the host is
+     * hard-rebooted instead. Before stopping any VM the fence arms a detached deadman that keeps running even if the
+     * agent kills the heartbeat script when {@link #HEARTBEAT_UPDATE_TIMEOUT} runs out (stopping a VM can block for a
+     * long time on hung storage). If the VMs are not all confirmed stopped within this budget, the deadman
+     * hard-reboots the host. Each virsh call is also limited to 10 seconds, or this value if lower, when coreutils
+     * {@code timeout} is installed. Keep it below {@link #HEARTBEAT_UPDATE_TIMEOUT}. A value that is not a positive
+     * integer falls back to the default. No effect on {@code hard-reboot} and {@code graceful-reboot}, or when
+     * {@link #KVM_HEARTBEAT_FENCE_KEEP_VMS_RUNNING} is true.<br>
+     * Read by the heartbeat shell scripts directly from agent.properties.<br>
+     * Data type: Integer.<br>
+     * Default value: <code>30</code>
+     */
+    public static final Property<Integer> KVM_HEARTBEAT_FENCE_STOP_VMS_TIMEOUT
+        = new Property<>("kvm.heartbeat.fence.stop.vms.timeout", 30);
+
+    /**
+     * Loop guard for the {@code restart-agent} fence action, in seconds. While storage keeps failing, the restarted
+     * agent runs the heartbeat again, it fails again and {@code restart-agent} would restart the agent again, over and
+     * over. Each {@code restart-agent} fence records its time under /run (kept across agent restarts, cleared by a
+     * reboot); if {@code restart-agent} already fired less than this many seconds ago, the fence hard-reboots the host
+     * instead. 0 turns the guard off. A value that is not a whole number of seconds falls back to the default. No
+     * effect on {@code hard-reboot}, {@code graceful-reboot} and {@code log-only}.<br>
+     * Read by the heartbeat shell scripts directly from agent.properties.<br>
+     * Data type: Integer.<br>
+     * Default value: <code>600</code>
+     */
+    public static final Property<Integer> KVM_HEARTBEAT_FENCE_RESTART_AGENT_COOLDOWN
+        = new Property<>("kvm.heartbeat.fence.restart.agent.cooldown", 600);
+
     /**
      * Enables manually setting CPU's topology on KVM's VM. <br>
      * Data type: Boolean.<br>