diff --git a/agent/conf/agent.properties b/agent/conf/agent.properties index 4e36eff4d75f..781602e4fd9d 100644 --- a/agent/conf/agent.properties +++ b/agent/conf/agent.properties @@ -318,6 +318,69 @@ iscsi.session.cleanup.enabled=false # This parameter specifies if the host must be rebooted when something goes wrong with the heartbeat. #reboot.host.and.alert.management.on.heartbeat.timeout=false +# Action taken by kvmheartbeat.sh / kvmspheartbeat.sh / kvmsmpheartbeat.sh when a storage heartbeat +# write fails persistently. Only consulted when fencing is enabled, i.e. +# 'reboot.host.and.alert.management.on.heartbeat.timeout' is true: that boolean remains +# the global on/off switch for fencing, this setting only chooses WHAT the fence does. +# +# Allowed values: +# hard-reboot - immediate sysrq-trigger reboot (default). +# Required default for setups where a stale NFSv3 mount can prevent +# a graceful shutdown from completing. +# graceful-reboot - 'systemctl reboot' instead of sysrq; allows VMs to stop cleanly. +# Use only if a stale storage mount cannot block shutdown. +# restart-agent - stop every VM on the host, then restart cloudstack-agent; the +# host itself stays up. +# log-only - stop every VM on the host, then only write a message to the +# host's syslog (no reboot, no agent restart, no management server +# alert). The operator must watch syslog and investigate. +# +# Any other value falls back to hard-reboot. +# +# restart-agent and log-only stop the VMs because, once the heartbeat is stale, the +# management server may declare the host Down and HA may start the same VMs on another +# host; two copies would then write to the same disks. If any VM cannot be stopped, +# the fence falls back to hard-reboot. +# +# The non-default values are recommended for setups using LINSTOR/DRBD or any local +# storage with replication, where transient I/O contention can cause a heartbeat +# write to time out without the host actually being unhealthy. +#kvm.heartbeat.fence.action=hard-reboot + +# Set to true to let restart-agent and log-only leave the VMs running. This is only +# safe when no VM on this host can be started on another host by HA (VM HA off for all +# of them and host HA not enabled). Otherwise a host the management server declares +# Down can end up with the same VM running twice and corrupting its disks. +# Has no effect on hard-reboot and graceful-reboot. +# This is the master opt-out: when true, nothing is stopped and +# kvm.heartbeat.fence.include.system.vms is ignored (the syslog warning says so). +#kvm.heartbeat.fence.keep.vms.running=false + +# Whether restart-agent and log-only also stop system VMs (virtual routers, console +# proxy, secondary storage and internal LB VMs, recognised by their r-, v-, s- and b- +# instance names). With true (default) every VM on the host is stopped. With false +# only user VMs are stopped; system VMs keep running and a system VM that is still +# running does not trigger the hard-reboot fallback. +# Risk of false: if the management server declares this host Down it restarts the +# system VMs on another host, so a virtual router could run twice with the same IP +# and MAC addresses on the guest network, and a second console proxy or secondary +# storage VM could run with the same addresses. Only set false if you accept that. +# Ignored when kvm.heartbeat.fence.keep.vms.running is true. Has no effect on +# hard-reboot and graceful-reboot. +#kvm.heartbeat.fence.include.system.vms=true + +# Seconds restart-agent and log-only may spend stopping VMs before the host is +# hard-rebooted instead. Before stopping any VM the fence arms a detached deadman that +# keeps running even if the agent kills the heartbeat script when +# heartbeat.update.timeout runs out (stopping a VM can block for a long time on hung +# storage). If the VMs are not all confirmed stopped within this budget, the deadman +# hard-reboots the host. Each virsh call is also limited to 10 seconds (or this value +# if lower) when coreutils timeout is installed. Keep it below heartbeat.update.timeout +# (60 seconds by default). A value that is not a positive integer means 30. +# Has no effect on hard-reboot and graceful-reboot, or when +# kvm.heartbeat.fence.keep.vms.running is true. +#kvm.heartbeat.fence.stop.vms.timeout=30 + # Enables manually setting CPU's topology on KVM's VM. #enable.manually.setting.cpu.topology.on.kvm.vm=true diff --git a/agent/src/main/java/com/cloud/agent/properties/AgentProperties.java b/agent/src/main/java/com/cloud/agent/properties/AgentProperties.java index e4775188d0ca..30e7e54aebc0 100644 --- a/agent/src/main/java/com/cloud/agent/properties/AgentProperties.java +++ b/agent/src/main/java/com/cloud/agent/properties/AgentProperties.java @@ -622,6 +622,80 @@ public class AgentProperties{ public static final Property REBOOT_HOST_AND_ALERT_MANAGEMENT_ON_HEARTBEAT_TIMEOUT = new Property<>("reboot.host.and.alert.management.on.heartbeat.timeout", false); + /** + * Action taken by the KVM agent's storage heartbeat scripts (kvmheartbeat.sh, kvmspheartbeat.sh + * and kvmsmpheartbeat.sh) when a heartbeat write fails persistently. Only consulted when + * {@link #REBOOT_HOST_AND_ALERT_MANAGEMENT_ON_HEARTBEAT_TIMEOUT} is true: that boolean remains the + * on/off switch for fencing, this setting only chooses what the fence does. Allowed values: + * + * Any other value falls back to {@code hard-reboot}.
+ * {@code restart-agent} and {@code log-only} stop the VMs because, once the heartbeat is stale, the management + * server may declare the host Down and HA may start the same VMs on another host, where both copies would write + * to the same disks. If any VM cannot be stopped the fence falls back to {@code hard-reboot}. See + * {@link #KVM_HEARTBEAT_FENCE_KEEP_VMS_RUNNING}.
+ * The non-default values are recommended for setups using LINSTOR/DRBD or other replicated + * local storage, where transient I/O contention can cause a heartbeat write to time out + * without the host actually being unhealthy.
+ * Read by the heartbeat shell scripts directly from agent.properties.
+ * Data type: String.
+ * Default value: {@code hard-reboot} + */ + public static final Property KVM_HEARTBEAT_FENCE_ACTION + = new Property<>("kvm.heartbeat.fence.action", "hard-reboot"); + + /** + * When true, the {@code restart-agent} and {@code log-only} fence actions leave the host's VMs running instead of + * stopping them first. Only safe when no VM on the host can be started on another host by HA (VM HA off for all + * of them and host HA not enabled); otherwise a host the management server declares Down can end up with the + * same VM running twice, writing to the same disks. No effect on {@code hard-reboot} and {@code graceful-reboot}. + * This is the master opt-out: when true nothing is stopped and {@link #KVM_HEARTBEAT_FENCE_INCLUDE_SYSTEM_VMS} + * is ignored.
+ * Read by the heartbeat shell scripts directly from agent.properties.
+ * Data type: Boolean.
+ * Default value: false + */ + public static final Property KVM_HEARTBEAT_FENCE_KEEP_VMS_RUNNING + = new Property<>("kvm.heartbeat.fence.keep.vms.running", false); + + /** + * Whether the {@code restart-agent} and {@code log-only} fence actions also stop system VMs (virtual routers, + * console proxy, secondary storage and internal LB VMs, recognised by their {@code r-}, {@code v-}, {@code s-} and + * {@code b-} instance names). When true every VM on the host is stopped. When false only user VMs are stopped; + * system VMs keep running and do not trigger the {@code hard-reboot} fallback. Risk of false: if the management + * server declares the host Down it restarts the system VMs on another host, so a virtual router could run twice + * with the same IP and MAC addresses. Ignored when {@link #KVM_HEARTBEAT_FENCE_KEEP_VMS_RUNNING} is true. No + * effect on {@code hard-reboot} and {@code graceful-reboot}. + * Read by the heartbeat shell scripts directly from agent.properties.
+ * Data type: Boolean.
+ * Default value: true + */ + public static final Property KVM_HEARTBEAT_FENCE_INCLUDE_SYSTEM_VMS + = new Property<>("kvm.heartbeat.fence.include.system.vms", true); + + /** + * Seconds the {@code restart-agent} and {@code log-only} fence actions may spend stopping VMs before the host is + * hard-rebooted instead. Before stopping any VM the fence arms a detached deadman that keeps running even if the + * agent kills the heartbeat script when {@link #HEARTBEAT_UPDATE_TIMEOUT} runs out (stopping a VM can block for a + * long time on hung storage). If the VMs are not all confirmed stopped within this budget, the deadman + * hard-reboots the host. Each virsh call is also limited to 10 seconds, or this value if lower, when coreutils + * {@code timeout} is installed. Keep it below {@link #HEARTBEAT_UPDATE_TIMEOUT}. A value that is not a positive + * integer falls back to the default. No effect on {@code hard-reboot} and {@code graceful-reboot}, or when + * {@link #KVM_HEARTBEAT_FENCE_KEEP_VMS_RUNNING} is true.
+ * Read by the heartbeat shell scripts directly from agent.properties.
+ * Data type: Integer.
+ * Default value: 30 + */ + public static final Property KVM_HEARTBEAT_FENCE_STOP_VMS_TIMEOUT + = new Property<>("kvm.heartbeat.fence.stop.vms.timeout", 30); + /** * Enables manually setting CPU's topology on KVM's VM.
* Data type: Boolean.
diff --git a/agent/src/test/java/com/cloud/agent/KvmHaFenceScriptTest.java b/agent/src/test/java/com/cloud/agent/KvmHaFenceScriptTest.java new file mode 100644 index 000000000000..0cc3ff962daa --- /dev/null +++ b/agent/src/test/java/com/cloud/agent/KvmHaFenceScriptTest.java @@ -0,0 +1,317 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.agent; + +import java.io.File; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.List; +import java.util.concurrent.TimeUnit; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import java.util.stream.Stream; + +import org.junit.After; +import org.junit.Assert; +import org.junit.Assume; +import org.junit.Before; +import org.junit.Test; + +/** + * Runs scripts/vm/hypervisor/kvm/kvmha-fence.sh with stand-ins for virsh, systemctl, logger and the sysrq trigger, + * so nothing on the machine running the test is stopped, restarted or rebooted. + */ +public class KvmHaFenceScriptTest { + + private static final File SCRIPT = new File("../scripts/vm/hypervisor/kvm/kvmha-fence.sh"); + + private Path dir; + private Path running; + private Path calls; + private Path syslog; + private Path sysrq; + private Path props; + private Path deadmanDir; + private Path hangPids; + + @Before + public void setUp() throws IOException { + Assume.assumeTrue("needs bash", new File("/bin/bash").canExecute()); + Assume.assumeTrue("needs the fence script", SCRIPT.isFile()); + dir = Files.createTempDirectory("kvmha-fence"); + running = dir.resolve("running"); + calls = dir.resolve("calls"); + syslog = dir.resolve("syslog"); + sysrq = dir.resolve("sysrq"); + props = dir.resolve("agent.properties"); + deadmanDir = dir.resolve("deadman"); + hangPids = dir.resolve("hang-pids"); + Files.createFile(calls); + Files.createFile(syslog); + stub("virsh", "echo \"virsh $*\" >> '" + calls + "'\n" + + "case \" $* \" in\n" + + " *' list '*) cat '" + running + "' ;;\n" + + " *' destroy '*) vm=\"${@: -1}\";" + + " if grep -qx \"$vm\" '" + dir.resolve("hang") + "' 2>/dev/null; then echo $$ >> '" + hangPids + "'; exec /bin/sleep 60; fi;" + + " grep -qx \"$vm\" '" + dir.resolve("stuck") + "' 2>/dev/null && exit 1;" + + " grep -vx \"$vm\" '" + running + "' > '" + running + ".new'; mv '" + running + ".new' '" + running + "' ;;\n" + + "esac\n"); + stub("systemctl", "echo \"systemctl $*\" >> '" + calls + "'\n"); + stub("logger", "echo \"$*\" >> '" + syslog + "'\n"); + // Fence delays are skipped; the deadman's 1 s steps stay real so its budget is measured in real time. + stub("sleep", "if [ \"$1\" = 1 ]; then exec /bin/sleep 1; fi\nexit 0\n"); + } + + @After + public void tearDown() throws IOException { + if (dir == null) { + return; + } + // Background processes the script may leave behind: armed deadmen and virsh stand-ins that hang. + final List pids = new ArrayList<>(deadmanPids()); + for (final String line : read(hangPids).split("\\s+")) { + if (!line.isEmpty()) { + pids.add(Long.parseLong(line)); + } + } + for (final long pid : pids) { + ProcessHandle.of(pid).ifPresent(ProcessHandle::destroyForcibly); + } + try (Stream paths = Files.walk(dir)) { + paths.sorted(Comparator.reverseOrder()).map(Path::toFile).forEach(File::delete); + } + } + + private void stub(final String name, final String body) throws IOException { + final Path file = dir.resolve(name); + Files.write(file, ("#!/bin/bash\n" + body).getBytes(StandardCharsets.UTF_8)); + Assert.assertTrue(file.toFile().setExecutable(true)); + } + + private int fence(final String properties, final String... runningVms) throws Exception { + final Process process = start(properties, runningVms); + Assert.assertTrue("fence script did not finish", process.waitFor(60, TimeUnit.SECONDS)); + return process.exitValue(); + } + + private Process start(final String properties, final String... runningVms) throws IOException { + Files.write(props, properties.getBytes(StandardCharsets.UTF_8)); + Files.write(running, (String.join("\n", runningVms) + "\n").getBytes(StandardCharsets.UTF_8)); + final ProcessBuilder pb = new ProcessBuilder("/bin/bash", "-c", "source \"$1\"; fence_action test.sh", "fence", + SCRIPT.getAbsolutePath()); + pb.environment().put("PATH", dir + File.pathSeparator + System.getenv("PATH")); + pb.environment().put("AGENT_PROPS", props.toString()); + pb.environment().put("FENCE_LOGGER", dir.resolve("logger").toString()); + pb.environment().put("FENCE_SYSRQ_TRIGGER", sysrq.toString()); + pb.environment().put("FENCE_DEADMAN_DIR", deadmanDir.toString()); + pb.redirectErrorStream(true); + pb.redirectOutput(dir.resolve("out").toFile()); + return pb.start(); + } + + /** Pids of every deadman the script started, as each one logs its own pid when it starts. */ + private List deadmanPids() throws IOException { + final List pids = new ArrayList<>(); + final Matcher m = Pattern.compile("fence deadman \\(pid (\\d+)\\) armed").matcher(read(syslog)); + while (m.find()) { + pids.add(Long.parseLong(m.group(1))); + } + return pids; + } + + private boolean waitFor(final long millis, final Check check) throws Exception { + final long deadline = System.currentTimeMillis() + millis; + while (System.currentTimeMillis() < deadline) { + if (check.ok()) { + return true; + } + Thread.sleep(100); + } + return check.ok(); + } + + private interface Check { + boolean ok() throws Exception; + } + + private String read(final Path file) throws IOException { + return file.toFile().exists() ? new String(Files.readAllBytes(file), StandardCharsets.UTF_8) : ""; + } + + private boolean rebooted() throws IOException { + return read(sysrq).trim().equals("b"); + } + + @Test + public void logOnlyStopsEveryVmAndDoesNotReboot() throws Exception { + Assert.assertEquals(0, fence("kvm.heartbeat.fence.action=log-only\n", "i-2-10-VM", "r-4-VM")); + Assert.assertTrue(read(calls).contains("destroy i-2-10-VM")); + Assert.assertTrue(read(calls).contains("destroy r-4-VM")); + Assert.assertEquals("", read(running).trim()); + Assert.assertFalse(read(calls).contains("systemctl")); + Assert.assertFalse(rebooted()); + } + + @Test + public void restartAgentStopsEveryVmBeforeRestartingTheAgent() throws Exception { + Assert.assertEquals(0, fence("kvm.heartbeat.fence.action=restart-agent\n", "i-2-10-VM")); + final String log = read(calls); + Assert.assertTrue(log.contains("destroy i-2-10-VM")); + Assert.assertTrue(log.contains("systemctl restart cloudstack-agent")); + Assert.assertTrue("VMs must be stopped before the agent restarts", + log.indexOf("destroy i-2-10-VM") < log.indexOf("systemctl restart cloudstack-agent")); + Assert.assertFalse(rebooted()); + } + + @Test + public void aVmThatCannotBeStoppedFallsBackToHardReboot() throws Exception { + Files.write(dir.resolve("stuck"), "i-2-11-VM\n".getBytes(StandardCharsets.UTF_8)); + fence("kvm.heartbeat.fence.action=restart-agent\n", "i-2-10-VM", "i-2-11-VM"); + Assert.assertTrue("a VM still running must not be left behind", rebooted()); + Assert.assertFalse(read(calls).contains("systemctl restart cloudstack-agent")); + Assert.assertTrue(read(syslog).contains("falling back to hard-reboot")); + } + + @Test + public void keepVmsRunningIsHonouredWithAWarning() throws Exception { + Assert.assertEquals(0, fence("kvm.heartbeat.fence.action=log-only\nkvm.heartbeat.fence.keep.vms.running=true\n", "i-2-10-VM")); + Assert.assertFalse(read(calls).contains("destroy")); + Assert.assertEquals("i-2-10-VM", read(running).trim()); + Assert.assertTrue(read(syslog).contains("WARNING")); + Assert.assertFalse(rebooted()); + } + + @Test + public void includeSystemVmsTrueStopsSystemVmsToo() throws Exception { + Assert.assertEquals(0, fence("kvm.heartbeat.fence.action=log-only\nkvm.heartbeat.fence.include.system.vms=true\n", + "i-2-10-VM", "r-4-VM", "s-1-VM", "v-2-VM", "b-5-VM")); + Assert.assertEquals("", read(running).trim()); + Assert.assertFalse(rebooted()); + } + + @Test + public void includeSystemVmsFalseStopsOnlyUserVms() throws Exception { + Assert.assertEquals(0, fence("kvm.heartbeat.fence.action=restart-agent\nkvm.heartbeat.fence.include.system.vms=false\n", + "i-2-10-VM", "r-4-VM", "r-7-VM-1234", "s-1-VM", "v-2-VM", "b-5-VM")); + final String log = read(calls); + Assert.assertTrue(log.contains("destroy i-2-10-VM")); + for (final String systemVm : new String[] {"r-4-VM", "r-7-VM-1234", "s-1-VM", "v-2-VM", "b-5-VM"}) { + Assert.assertFalse(systemVm + " must keep running", log.contains("destroy " + systemVm)); + } + Assert.assertEquals("r-4-VM\nr-7-VM-1234\ns-1-VM\nv-2-VM\nb-5-VM", read(running).trim()); + Assert.assertTrue(log.contains("systemctl restart cloudstack-agent")); + Assert.assertTrue(read(syslog).contains("include.system.vms=false")); + Assert.assertFalse(rebooted()); + } + + @Test + public void includeSystemVmsFalseIgnoresAStuckSystemVmButNotAStuckUserVm() throws Exception { + Files.write(dir.resolve("stuck"), "r-4-VM\n".getBytes(StandardCharsets.UTF_8)); + Assert.assertEquals(0, fence("kvm.heartbeat.fence.action=log-only\nkvm.heartbeat.fence.include.system.vms=false\n", + "i-2-10-VM", "r-4-VM")); + Assert.assertFalse("a running system VM must not force a reboot", rebooted()); + + Files.write(dir.resolve("stuck"), "i-2-11-VM\n".getBytes(StandardCharsets.UTF_8)); + fence("kvm.heartbeat.fence.action=log-only\nkvm.heartbeat.fence.include.system.vms=false\n", "i-2-11-VM", "r-4-VM"); + Assert.assertTrue("a user VM still running must still force a reboot", rebooted()); + } + + @Test + public void keepVmsRunningWinsOverIncludeSystemVms() throws Exception { + Assert.assertEquals(0, fence("kvm.heartbeat.fence.action=log-only\nkvm.heartbeat.fence.keep.vms.running=true\n" + + "kvm.heartbeat.fence.include.system.vms=false\n", "i-2-10-VM", "r-4-VM")); + Assert.assertFalse(read(calls).contains("destroy")); + Assert.assertEquals("i-2-10-VM\nr-4-VM", read(running).trim()); + Assert.assertTrue(read(syslog).contains("overrides kvm.heartbeat.fence.include.system.vms")); + Assert.assertFalse(rebooted()); + } + + @Test + public void hardRebootIsTheDefaultAndDoesNotTouchVmsFirst() throws Exception { + fence("", "i-2-10-VM"); + Assert.assertTrue(rebooted()); + Assert.assertFalse(read(calls).contains("destroy")); + } + + @Test + public void unknownActionFallsBackToHardReboot() throws Exception { + fence("kvm.heartbeat.fence.action=custom\n", "i-2-10-VM"); + Assert.assertTrue(rebooted()); + } + + @Test + public void deadmanRebootsWhenTheAgentKillsTheScriptWhileAVmStopHangs() throws Exception { + Files.write(dir.resolve("hang"), "i-2-12-VM\n".getBytes(StandardCharsets.UTF_8)); + final Process process = start("kvm.heartbeat.fence.action=restart-agent\nkvm.heartbeat.fence.stop.vms.timeout=2\n", + "i-2-12-VM"); + Assert.assertTrue("virsh destroy never started", waitFor(5000, () -> !read(hangPids).trim().isEmpty())); + Thread.sleep(500); + // What the agent does when heartbeat.update.timeout runs out (Script.execute). + process.destroyForcibly(); + Assert.assertTrue(process.waitFor(5, TimeUnit.SECONDS)); + Assert.assertFalse("the script itself must not have reached its fallback yet", rebooted()); + Assert.assertTrue("the deadman must hard-reboot the host after the script was killed", + waitFor(6000, this::rebooted)); + Assert.assertTrue(read(syslog).contains("fence deadman is hard-rebooting the host")); + Assert.assertFalse(read(calls).contains("systemctl restart cloudstack-agent")); + } + + @Test + public void deadmanIsDisarmedWhenEveryVmStopsWithinTheBudget() throws Exception { + Assert.assertEquals(0, fence("kvm.heartbeat.fence.action=log-only\nkvm.heartbeat.fence.stop.vms.timeout=2\n", + "i-2-10-VM", "r-4-VM")); + Assert.assertEquals("", read(running).trim()); + Assert.assertTrue("the deadman must have been armed", deadmanDir.toFile().isDirectory()); + Assert.assertTrue("arm and pid files must be gone", waitFor(3000, () -> { + final String[] left = deadmanDir.toFile().list(); + return left != null && left.length == 0; + })); + for (final long pid : deadmanPids()) { + Assert.assertTrue("deadman " + pid + " must not be left running", + waitFor(3000, () -> !ProcessHandle.of(pid).map(ProcessHandle::isAlive).orElse(false))); + } + // Past the 2 s budget nothing may reboot the host. + Thread.sleep(3000); + Assert.assertFalse(rebooted()); + } + + @Test + public void invalidStopBudgetFallsBackToTheDefault() throws Exception { + for (final String value : new String[] {"abc", "0", "-5", "1.5"}) { + Files.write(syslog, new byte[0]); + Assert.assertEquals(0, fence("kvm.heartbeat.fence.action=log-only\nkvm.heartbeat.fence.stop.vms.timeout=" + value + "\n", + "i-2-10-VM")); + final String log = read(syslog); + Assert.assertTrue(value + " must be reported", log.contains("kvm.heartbeat.fence.stop.vms.timeout='" + value + "'")); + Assert.assertTrue(value + " must give the default budget", log.contains("with a budget of 30 s")); + Assert.assertTrue(log.contains("at most 10 s per virsh call")); + Assert.assertFalse(rebooted()); + } + } + + @Test + public void unsetStopBudgetUsesTheDefaultWithoutAWarning() throws Exception { + Assert.assertEquals(0, fence("kvm.heartbeat.fence.action=log-only\n", "i-2-10-VM")); + final String log = read(syslog); + Assert.assertTrue(log.contains("with a budget of 30 s")); + Assert.assertFalse(log.contains("is not a positive number")); + } +} diff --git a/scripts/vm/hypervisor/kvm/kvmha-fence.sh b/scripts/vm/hypervisor/kvm/kvmha-fence.sh new file mode 100755 index 000000000000..77c582eb5fdb --- /dev/null +++ b/scripts/vm/hypervisor/kvm/kvmha-fence.sh @@ -0,0 +1,263 @@ +#!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# Shared fence-action helper for kvmheartbeat.sh, kvmspheartbeat.sh and kvmsmpheartbeat.sh. +# Sourced by those scripts; do not invoke directly. +# +# kvm.heartbeat.fence.action in agent.properties selects the action: +# hard-reboot (default), graceful-reboot, restart-agent, or log-only (only writes to +# syslog). Any other value falls back to hard-reboot. +# +# restart-agent and log-only leave the host up. Once the heartbeat is stale the management +# server may declare this host Down and HA may start its VMs on another host, so both +# actions first stop every VM still running here; if any VM cannot be stopped the fence +# falls back to hard-reboot. kvm.heartbeat.fence.include.system.vms=false limits that step +# to user VMs: system VMs (virtual routers, console proxy, secondary storage and internal +# LB VMs) are left running and do not count towards the fallback. +# kvm.heartbeat.fence.keep.vms.running=true skips the step entirely and wins over +# include.system.vms; it is only safe when no VM on this host can be restarted elsewhere. +# +# The agent kills this script once heartbeat.update.timeout (60 s by default) runs out, and +# on hung storage stopping a VM can block for much longer than that. So before stopping any +# VM the fence arms a detached deadman that survives the agent killing this script: unless +# every VM the fence must stop is confirmed gone within kvm.heartbeat.fence.stop.vms.timeout +# seconds (default 30, keep it below heartbeat.update.timeout), the deadman hard-reboots the +# host. Each virsh call is also bounded by coreutils timeout, when present, to at most 10 s +# (or the budget if lower) so one stuck domain cannot use up the whole budget. +# +# Properties read from agent.properties: +# kvm.heartbeat.fence.action hard-reboot (default), graceful-reboot, +# restart-agent or log-only +# kvm.heartbeat.fence.keep.vms.running false (default); true stops nothing +# kvm.heartbeat.fence.include.system.vms true (default); false stops only user VMs +# kvm.heartbeat.fence.stop.vms.timeout 30 (default); seconds the VM stop phase may take +# before the deadman hard-reboots the host. A value +# that is not a positive integer means 30. +# +# Usage from caller: +# source "$(dirname "$0")/kvmha-fence.sh" +# fence_action "kvmheartbeat.sh" # script name passed for log tagging + +AGENT_PROPS="${AGENT_PROPS:-/etc/cloudstack/agent/agent.properties}" +FENCE_LOGGER="${FENCE_LOGGER:-/usr/bin/logger}" +FENCE_SYSRQ_TRIGGER="${FENCE_SYSRQ_TRIGGER:-/proc/sysrq-trigger}" +# Where the deadman keeps its arm file; /run is a local tmpfs, so hung storage cannot block it. +FENCE_DEADMAN_DIR="${FENCE_DEADMAN_DIR:-/run/cloudstack-agent/kvmha-fence}" +FENCE_STOP_VMS_TIMEOUT_DEFAULT=30 +FENCE_VIRSH_TIMEOUT_MAX=10 +# Per-call virsh timeout in seconds; empty means unbounded. Set by fence_action. +FENCE_VIRSH_TIMEOUT="" +FENCE_DEADMAN_ARM="" + +read_agent_property() { + [ -r "$AGENT_PROPS" ] || return 0 + grep "^$1=" "$AGENT_PROPS" | tail -n 1 | cut -d= -f2- | tr -d '[:space:]' +} + +# Prints the VM stop budget in seconds from kvm.heartbeat.fence.stop.vms.timeout. Unset gives +# the default; anything else that is not a positive integer (up to 6 digits) gives the default +# with a warning in syslog. +stop_vms_budget() { + local source_script="$1" val + val=$(read_agent_property "kvm.heartbeat.fence.stop.vms.timeout") + if [[ "$val" =~ ^[0-9]{1,6}$ ]] && [ $((10#$val)) -gt 0 ]; then + echo $((10#$val)) + return 0 + fi + if [ -n "$val" ]; then + "$FENCE_LOGGER" -t heartbeat "${source_script}: WARNING kvm.heartbeat.fence.stop.vms.timeout='${val}' is not a positive number of seconds; using the default of ${FENCE_STOP_VMS_TIMEOUT_DEFAULT} s." + fi + echo "$FENCE_STOP_VMS_TIMEOUT_DEFAULT" +} + +# Runs virsh against the system libvirt, bounded by FENCE_VIRSH_TIMEOUT seconds when coreutils +# timeout is available. Without timeout the call is unbounded and only the deadman covers it. +fence_virsh() { + if [ -n "$FENCE_VIRSH_TIMEOUT" ] && command -v timeout > /dev/null 2>&1; then + timeout -k 2 "$FENCE_VIRSH_TIMEOUT" virsh -c qemu:///system "$@" + else + virsh -c qemu:///system "$@" + fi +} + +# Body of the deadman; runs in its own detached bash (see arm_deadman), never in the fence +# script itself. Waits up to budget seconds in 1 s steps and exits quietly as soon as the arm +# file is removed. If the arm file is still there when the budget runs out, the VM stop phase +# did not finish in time (or the agent killed the fence script), so it hard-reboots the host. +fence_deadman() { + local arm="$1" budget="$2" source_script="$3" + local deadline=$((SECONDS + budget)) + # Survive the agent or a service stop sending TERM or HUP; only disarm_deadman stops it early. + trap '' HUP INT TERM + echo "$$" > "${arm}.pid" + trap 'rm -f "${arm}.pid"' EXIT + "$FENCE_LOGGER" -t heartbeat "${source_script}: fence deadman (pid $$) armed; the host will be hard-rebooted in ${budget} s unless every VM the fence must stop is confirmed stopped." + while [ "$SECONDS" -lt "$deadline" ]; do + [ -e "$arm" ] || exit 0 + sleep 1 + done + [ -e "$arm" ] || exit 0 + "$FENCE_LOGGER" -t heartbeat "${source_script}: VMs were not confirmed stopped within kvm.heartbeat.fence.stop.vms.timeout (${budget} s); fence deadman is hard-rebooting the host so no VM keeps running while HA may start it elsewhere." + sync & + sleep 5 + echo b > "$FENCE_SYSRQ_TRIGGER" +} + +# Starts the deadman for a budget of $2 seconds, detached from this script (its own session +# when setsid exists) so it keeps running if the agent kills the fence script on timeout. +# Fails if it cannot be armed. +arm_deadman() { + local source_script="$1" budget="$2" arm + # The deadman runs a new bash, so it gets the values it needs passed in explicitly. + # shellcheck disable=SC2016 + local body='source "$1"; fence_deadman "$2" "$3" "$4"' + mkdir -p "$FENCE_DEADMAN_DIR" 2> /dev/null || return 1 + arm="${FENCE_DEADMAN_DIR}/armed.$$" + : > "$arm" 2> /dev/null || return 1 + FENCE_DEADMAN_ARM="$arm" + if command -v setsid > /dev/null 2>&1; then + FENCE_LOGGER="$FENCE_LOGGER" FENCE_SYSRQ_TRIGGER="$FENCE_SYSRQ_TRIGGER" \ + setsid nohup /bin/bash -c "$body" fence-deadman "${BASH_SOURCE[0]}" "$arm" "$budget" "$source_script" \ + < /dev/null > /dev/null 2>&1 & + else + FENCE_LOGGER="$FENCE_LOGGER" FENCE_SYSRQ_TRIGGER="$FENCE_SYSRQ_TRIGGER" \ + nohup /bin/bash -c "$body" fence-deadman "${BASH_SOURCE[0]}" "$arm" "$budget" "$source_script" \ + < /dev/null > /dev/null 2>&1 & + fi + disown 2> /dev/null || true +} + +# Stops the deadman: removing the arm file is enough on its own (it checks every second), +# killing it just makes it go away at once. +disarm_deadman() { + local arm="$FENCE_DEADMAN_ARM" pid + [ -n "$arm" ] || return 0 + rm -f "$arm" + pid=$(cat "${arm}.pid" 2> /dev/null) + if [[ "$pid" =~ ^[0-9]+$ ]]; then + kill -KILL "$pid" 2> /dev/null + fi + rm -f "${arm}.pid" + FENCE_DEADMAN_ARM="" +} + +# True when the libvirt domain name is a CloudStack system VM, using the names the management +# server gives them (com.cloud.vm.VirtualMachineName): r--[-] for virtual +# routers, and s-, v- and b-- for secondary storage, console proxy and internal +# LB VMs. User VMs are i--- and never match. +is_system_vm() { + [[ "$1" =~ ^r-[0-9]+-[^-]+(-[^-]+)?$ || "$1" =~ ^[svb]-[0-9]+-[^-]+$ ]] +} + +# Prints the active VMs the fence has to stop: every VM, or only user VMs when +# include_system_vms is not "true". Fails if libvirt cannot be queried. +vms_to_stop() { + local include_system_vms="$1" + local vms vm + vms=$(fence_virsh list --name 2>/dev/null) || return 1 + for vm in $vms; do + if [ "$include_system_vms" = "true" ] || ! is_system_vm "$vm"; then + echo "$vm" + fi + done +} + +# Stops the active VMs selected by vms_to_stop. Fails if libvirt cannot be queried or any of +# them is still active afterwards, so the caller never assumes the VMs are stopped when they +# are not. System VMs left running on purpose do not make it fail. +stop_vms() { + local source_script="$1" + local include_system_vms="$2" + local vms vm + vms=$(vms_to_stop "$include_system_vms") || return 1 + for vm in $vms; do + "$FENCE_LOGGER" -t heartbeat "${source_script}: stopping VM ${vm} so it cannot run here while HA starts it on another host." + fence_virsh destroy "$vm" > /dev/null 2>&1 + done + vms=$(vms_to_stop "$include_system_vms") || return 1 + [ -z "$(echo "$vms" | tr -d '[:space:]')" ] +} + +fence_action() { + local source_script="${1:-kvmha}" + local FENCE_ACTION="hard-reboot" + local val + val=$(read_agent_property "kvm.heartbeat.fence.action") + [ -n "$val" ] && FENCE_ACTION="$val" + + case "$FENCE_ACTION" in + log-only|restart-agent) + local keep_running include_system_vms="true" + keep_running=$(read_agent_property "kvm.heartbeat.fence.keep.vms.running") + [ "$(read_agent_property "kvm.heartbeat.fence.include.system.vms")" = "false" ] && include_system_vms="false" + if [ "$keep_running" = "true" ]; then + # keep.vms.running is the master opt-out and wins over include.system.vms. + "$FENCE_LOGGER" -t heartbeat "${source_script}: WARNING fence action '${FENCE_ACTION}' with kvm.heartbeat.fence.keep.vms.running=true leaves every VM on this host running, system VMs included (this overrides kvm.heartbeat.fence.include.system.vms). If the management server marks this host Down, HA may start the same VMs on another host and both copies would write to the same disks." + else + if [ "$include_system_vms" = "false" ]; then + "$FENCE_LOGGER" -t heartbeat "${source_script}: WARNING fence action '${FENCE_ACTION}' with kvm.heartbeat.fence.include.system.vms=false stops only user VMs and leaves system VMs running. If the management server marks this host Down it may start those system VMs on another host, and a virtual router could then run twice with the same IP and MAC addresses." + fi + local budget + budget=$(stop_vms_budget "$source_script") + FENCE_VIRSH_TIMEOUT=$(( budget < FENCE_VIRSH_TIMEOUT_MAX ? budget : FENCE_VIRSH_TIMEOUT_MAX )) + "$FENCE_LOGGER" -t heartbeat "${source_script}: stopping VMs for fence action '${FENCE_ACTION}' with a budget of ${budget} s (kvm.heartbeat.fence.stop.vms.timeout) and at most ${FENCE_VIRSH_TIMEOUT} s per virsh call." + if ! arm_deadman "$source_script" "$budget"; then + "$FENCE_LOGGER" -t heartbeat "${source_script}: could not arm the fence deadman in ${FENCE_DEADMAN_DIR}; falling back to hard-reboot because the VM stop phase could not be bounded." + FENCE_ACTION="hard-reboot" + elif ! stop_vms "$source_script" "$include_system_vms"; then + # The deadman stays armed: if this script is killed before it reboots, the deadman still does. + "$FENCE_LOGGER" -t heartbeat "${source_script}: could not stop every VM on this host that the fence must stop for action '${FENCE_ACTION}'; falling back to hard-reboot so no VM keeps running while HA may start it elsewhere." + FENCE_ACTION="hard-reboot" + else + # Every selected VM is confirmed gone, so the host may stay up. + disarm_deadman + fi + fi + ;; + esac + + case "$FENCE_ACTION" in + log-only) + # No reboot, no agent restart and no management server alert. + "$FENCE_LOGGER" -t heartbeat "${source_script}: heartbeat write to storage failed; fence action 'log-only' selected, taking no further action. Operator must investigate." + exit 0 + ;; + restart-agent) + "$FENCE_LOGGER" -t heartbeat "${source_script}: heartbeat write to storage failed; fence action 'restart-agent', restarting cloudstack-agent." + sync & + sleep 2 + systemctl restart cloudstack-agent + exit $? + ;; + graceful-reboot) + "$FENCE_LOGGER" -t heartbeat "${source_script}: heartbeat write to storage failed; fence action 'graceful-reboot', rebooting via systemctl (allows running VMs to stop cleanly)." + sync & + sleep 5 + systemctl reboot + exit $? + ;; + hard-reboot|*) + # Default, and the fallback for any unrecognised value: identical to the pre-existing behaviour. + "$FENCE_LOGGER" -t heartbeat "${source_script} will reboot system because it was unable to write the heartbeat to the storage." + sync & + sleep 5 + echo b > "$FENCE_SYSRQ_TRIGGER" + exit $? + ;; + esac +} diff --git a/scripts/vm/hypervisor/kvm/kvmheartbeat.sh b/scripts/vm/hypervisor/kvm/kvmheartbeat.sh index 1fa49b807769..645d70dc9ad9 100755 --- a/scripts/vm/hypervisor/kvm/kvmheartbeat.sh +++ b/scripts/vm/hypervisor/kvm/kvmheartbeat.sh @@ -156,11 +156,9 @@ then exit 0 elif [ "$cflag" == "1" ] then - /usr/bin/logger -t heartbeat "kvmheartbeat.sh will reboot system because it was unable to write the heartbeat to the storage." - sync & - sleep 5 - echo b > /proc/sysrq-trigger - exit $? + # shellcheck disable=SC1091 + . "$(dirname "$0")/kvmha-fence.sh" + fence_action "kvmheartbeat.sh" else write_hbLog exit $? diff --git a/scripts/vm/hypervisor/kvm/kvmsmpheartbeat.sh b/scripts/vm/hypervisor/kvm/kvmsmpheartbeat.sh index b102a1a866bb..b5dddefe1a9c 100755 --- a/scripts/vm/hypervisor/kvm/kvmsmpheartbeat.sh +++ b/scripts/vm/hypervisor/kvm/kvmsmpheartbeat.sh @@ -207,11 +207,13 @@ then exit 0 elif [ "$cflag" == "1" ] then - /usr/bin/logger -t heartbeat "kvmsmpheartbeat.sh will reboot system because it was unable to write the heartbeat to the storage." - sync & - sleep 5 - echo b > /proc/sysrq-trigger - exit $? + # Delegate the fence action to the shared helper so this script honours the + # kvm.heartbeat.fence.action property (hard-reboot / graceful-reboot / + # restart-agent / log-only) like kvmheartbeat.sh and kvmspheartbeat.sh. + # Default (no property set) remains hard-reboot via sysrq-trigger. + # shellcheck disable=SC1091 + . "$(dirname "$0")/kvmha-fence.sh" + fence_action "kvmsmpheartbeat.sh" else write_hbLog exit $? diff --git a/scripts/vm/hypervisor/kvm/kvmspheartbeat.sh b/scripts/vm/hypervisor/kvm/kvmspheartbeat.sh index 3cb459e3e854..1a5b2a24dd9f 100755 --- a/scripts/vm/hypervisor/kvm/kvmspheartbeat.sh +++ b/scripts/vm/hypervisor/kvm/kvmspheartbeat.sh @@ -58,9 +58,7 @@ deleteVMs() { if [ "$cflag" == "1" ] then - /usr/bin/logger -t heartbeat "kvmspheartbeat.sh will reboot system because it was unable to write the heartbeat to the storage." - sync & - sleep 5 - echo b > /proc/sysrq-trigger - exit $? + # shellcheck disable=SC1091 + . "$(dirname "$0")/kvmha-fence.sh" + fence_action "kvmspheartbeat.sh" fi