From 19523e36b617bbdfd56ec78977b0b2a45bb5c471 Mon Sep 17 00:00:00 2001 From: calvix <7136358+calvix@users.noreply.github.com> Date: Mon, 19 Oct 2026 08:40:15 +0200 Subject: [PATCH] vr: serve UDP load balancer rules with IPVS haproxy cannot balance UDP, so the virtual router refused UDP load balancer rules although it lists udp as a supported protocol. UDP rules are now left out of the haproxy configuration and sent to the router as a separate list. The router serves them with IPVS in NAT mode (it is the gateway of the backends), keeping the IPVS table in sync with the rules instead of flushing it, and opens the UDP port in its firewall for the rule's source CIDRs. - algorithms map to IPVS schedulers: roundrobin -> rr, leastconn -> lc, source -> sh; source based stickiness becomes IPVS persistence with its expire value as the timeout - other stickiness methods and health check policies are refused for UDP rules - a UDP rule gets a UDP firewall rule, and its protocol cannot be switched between udp and the tcp based protocols --- .../CreateLoadBalancerRuleCmd.java | 4 + .../facade/LoadBalancerConfigItem.java | 1 + .../model/LoadBalancerRule.java | 53 +++++++++++++ .../cloud/network/HAProxyConfigurator.java | 75 ++++++++++++++++++- .../network/LoadBalancerConfigurator.java | 3 + .../network/HAProxyConfiguratorTest.java | 62 +++++++++++++++ .../lb/LoadBalancingRulesManagerImpl.java | 4 + .../network/router/NetworkHelperImpl.java | 32 +++++++- .../network/router/NetworkHelperImplTest.java | 44 +++++++++++ .../debian/opt/cloud/bin/cs/CsLoadBalancer.py | 70 +++++++++++++++++ 10 files changed, 343 insertions(+), 5 deletions(-) diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/loadbalancer/CreateLoadBalancerRuleCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/loadbalancer/CreateLoadBalancerRuleCmd.java index bd72f248364e..3d2058c523a3 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/loadbalancer/CreateLoadBalancerRuleCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/loadbalancer/CreateLoadBalancerRuleCmd.java @@ -327,6 +327,10 @@ public Integer getSourcePortEnd() { } public String getProtocol() { + // a UDP load balancer needs a UDP firewall rule and must not clash with UDP port forwarding on the same port + if (NetUtils.UDP_PROTO.equals(getLbProtocol())) { + return NetUtils.UDP_PROTO; + } return NetUtils.TCP_PROTO; } diff --git a/core/src/main/java/com/cloud/agent/resource/virtualnetwork/facade/LoadBalancerConfigItem.java b/core/src/main/java/com/cloud/agent/resource/virtualnetwork/facade/LoadBalancerConfigItem.java index 6dae886b4137..71e68deba611 100644 --- a/core/src/main/java/com/cloud/agent/resource/virtualnetwork/facade/LoadBalancerConfigItem.java +++ b/core/src/main/java/com/cloud/agent/resource/virtualnetwork/facade/LoadBalancerConfigItem.java @@ -58,6 +58,7 @@ public List generateConfig(final NetworkElementCommand cmd) { final LoadBalancerRule loadBalancerRule = new LoadBalancerRule(configuration, tmpCfgFilePath, tmpCfgFileName, addRules, removeRules, statRules, routerIp); final LoadBalancerRule.SslCertEntry[] sslCerts = cfgtr.generateSslCertEntries(command); loadBalancerRule.setSslCerts(sslCerts); + loadBalancerRule.setUdpRules(cfgtr.generateUdpEntries(command)); final List rules = new LinkedList(); rules.add(loadBalancerRule); diff --git a/core/src/main/java/com/cloud/agent/resource/virtualnetwork/model/LoadBalancerRule.java b/core/src/main/java/com/cloud/agent/resource/virtualnetwork/model/LoadBalancerRule.java index 361c4765cc52..1e6623f94719 100644 --- a/core/src/main/java/com/cloud/agent/resource/virtualnetwork/model/LoadBalancerRule.java +++ b/core/src/main/java/com/cloud/agent/resource/virtualnetwork/model/LoadBalancerRule.java @@ -26,6 +26,7 @@ public class LoadBalancerRule { private String tmpCfgFilePath; private String tmpCfgFileName; private SslCertEntry[] sslCerts; + private UdpLbEntry[] udpRules; private String[] addRules; private String[] removeRules; @@ -80,6 +81,50 @@ public String getPassword() { } } + /** + * A UDP load balancer rule. The router serves it with IPVS (NAT mode), not haproxy. + */ + public static class UdpLbEntry { + private String sourceIp; + private int sourcePort; + // IPVS scheduler: rr, lc or sh + private String scheduler; + // IPVS persistence timeout in seconds, 0 = none + private int persistence; + // "ip:port" of each backend + private String[] destinations; + // allowed source CIDRs, empty = everyone + private String[] cidrs; + + public UdpLbEntry(String sourceIp, int sourcePort, String scheduler, int persistence, String[] destinations, String[] cidrs) { + this.sourceIp = sourceIp; + this.sourcePort = sourcePort; + this.scheduler = scheduler; + this.persistence = persistence; + this.destinations = destinations; + this.cidrs = cidrs; + } + + public String getSourceIp() { + return sourceIp; + } + public int getSourcePort() { + return sourcePort; + } + public String getScheduler() { + return scheduler; + } + public int getPersistence() { + return persistence; + } + public String[] getDestinations() { + return destinations; + } + public String[] getCidrs() { + return cidrs; + } + } + public LoadBalancerRule() { // Empty constructor for (de)serialization } @@ -157,4 +202,12 @@ public SslCertEntry[] getSslCerts() { public void setSslCerts(final SslCertEntry[] sslCerts) { this.sslCerts = sslCerts; } + + public UdpLbEntry[] getUdpRules() { + return udpRules; + } + + public void setUdpRules(final UdpLbEntry[] udpRules) { + this.udpRules = udpRules; + } } diff --git a/core/src/main/java/com/cloud/network/HAProxyConfigurator.java b/core/src/main/java/com/cloud/network/HAProxyConfigurator.java index 7d544c2e49c0..d1b9b8ce046d 100644 --- a/core/src/main/java/com/cloud/network/HAProxyConfigurator.java +++ b/core/src/main/java/com/cloud/network/HAProxyConfigurator.java @@ -37,6 +37,7 @@ import com.cloud.agent.api.to.LoadBalancerTO.StickinessPolicyTO; import com.cloud.agent.api.to.PortForwardingRuleTO; import com.cloud.agent.resource.virtualnetwork.model.LoadBalancerRule.SslCertEntry; +import com.cloud.agent.resource.virtualnetwork.model.LoadBalancerRule.UdpLbEntry; import com.cloud.network.lb.LoadBalancingRule.LbSslCert; import com.cloud.network.rules.LbStickinessMethod.StickinessMethodType; import com.cloud.utils.Pair; @@ -683,7 +684,7 @@ public String[] generateConfiguration(final LoadBalancerConfigCommand lbCmd) { result.add(blankLine); boolean has_listener = false; for (final LoadBalancerTO lbTO : lbCmd.getLoadBalancers()) { - if (lbTO.isRevoked()) { + if (lbTO.isRevoked() || isUdp(lbTO)) { continue; } final List poolRules = getRulesForPool(lbTO, lbCmd); @@ -708,6 +709,9 @@ public String[][] generateFwRules(final LoadBalancerConfigCommand lbCmd) { final Set toStats = new HashSet(); for (final LoadBalancerTO lbTO : lbCmd.getLoadBalancers()) { + if (isUdp(lbTO)) { + continue; + } final StringBuilder sb = new StringBuilder(); sb.append(lbTO.getSrcIp()).append(":"); @@ -764,4 +768,73 @@ private void addSslCertEntry(Set sslCertEntries, LoadBalancerTO lb final SslCertEntry sslCertEntry = new SslCertEntry(name, cert.getCert(), cert.getKey(), cert.getChain(), cert.getPassword()); sslCertEntries.add(sslCertEntry); } + + private static boolean isUdp(final LoadBalancerTO lbTO) { + return NetUtils.UDP_PROTO.equalsIgnoreCase(lbTO.getLbProtocol()); + } + + @Override + public UdpLbEntry[] generateUdpEntries(final LoadBalancerConfigCommand lbCmd) { + final List result = new ArrayList<>(); + for (final LoadBalancerTO lbTO : lbCmd.getLoadBalancers()) { + if (lbTO.isRevoked() || !isUdp(lbTO)) { + continue; + } + final List destinations = new ArrayList<>(); + for (final DestinationTO dest : lbTO.getDestinations()) { + if (!dest.isRevoked()) { + destinations.add(dest.getDestIp() + ":" + dest.getDestPort()); + } + } + final String[] cidrs = StringUtils.isBlank(lbTO.getCidrList()) ? new String[0] : lbTO.getCidrList().trim().split("\\s+"); + result.add(new UdpLbEntry(lbTO.getSrcIp(), lbTO.getSrcPort(), getIpvsScheduler(lbTO.getAlgorithm()), getIpvsPersistence(lbTO), + destinations.toArray(new String[0]), cidrs)); + } + return result.toArray(new UdpLbEntry[0]); + } + + protected static String getIpvsScheduler(final String algorithm) { + if ("leastconn".equals(algorithm)) { + return "lc"; + } + if ("source".equals(algorithm)) { + return "sh"; + } + return "rr"; + } + + /** + * Source based stickiness becomes IPVS persistence; its "expire" value (default 3h, as for haproxy) is the timeout. + */ + protected static int getIpvsPersistence(final LoadBalancerTO lbTO) { + if (lbTO.getStickinessPolicies() == null) { + return 0; + } + for (final StickinessPolicyTO policy : lbTO.getStickinessPolicies()) { + if (policy == null || !StickinessMethodType.SourceBased.getName().equalsIgnoreCase(policy.getMethodName())) { + continue; + } + String expire = "3h"; + for (final Pair param : policy.getParams()) { + if ("expire".equalsIgnoreCase(param.first()) && StringUtils.isNotBlank(param.second())) { + expire = param.second().trim(); + } + } + return durationToSeconds(expire); + } + return 0; + } + + private static int durationToSeconds(final String duration) { + final char unit = duration.charAt(duration.length() - 1); + final int multiplier; + switch (unit) { + case 's': multiplier = 1; break; + case 'm': multiplier = 60; break; + case 'h': multiplier = 3600; break; + case 'd': multiplier = 86400; break; + default: return Integer.parseInt(duration); + } + return Integer.parseInt(duration.substring(0, duration.length() - 1)) * multiplier; + } } diff --git a/core/src/main/java/com/cloud/network/LoadBalancerConfigurator.java b/core/src/main/java/com/cloud/network/LoadBalancerConfigurator.java index 8814f60b0714..839f94dcd486 100644 --- a/core/src/main/java/com/cloud/network/LoadBalancerConfigurator.java +++ b/core/src/main/java/com/cloud/network/LoadBalancerConfigurator.java @@ -24,6 +24,7 @@ import com.cloud.agent.api.routing.LoadBalancerConfigCommand; import com.cloud.agent.api.to.PortForwardingRuleTO; import com.cloud.agent.resource.virtualnetwork.model.LoadBalancerRule.SslCertEntry; +import com.cloud.agent.resource.virtualnetwork.model.LoadBalancerRule.UdpLbEntry; public interface LoadBalancerConfigurator { public final static int ADD = 0; @@ -37,4 +38,6 @@ public interface LoadBalancerConfigurator { public String[][] generateFwRules(LoadBalancerConfigCommand lbCmd); public SslCertEntry[] generateSslCertEntries(LoadBalancerConfigCommand lbCmd); + + public UdpLbEntry[] generateUdpEntries(LoadBalancerConfigCommand lbCmd); } diff --git a/core/src/test/java/com/cloud/network/HAProxyConfiguratorTest.java b/core/src/test/java/com/cloud/network/HAProxyConfiguratorTest.java index 073f976719b5..0709a967c7da 100644 --- a/core/src/test/java/com/cloud/network/HAProxyConfiguratorTest.java +++ b/core/src/test/java/com/cloud/network/HAProxyConfiguratorTest.java @@ -32,6 +32,9 @@ import com.cloud.agent.api.to.LoadBalancerTO; import com.cloud.network.lb.LoadBalancingRule.LbDestination; import com.cloud.network.lb.LoadBalancingRule.LbSslCert; +import com.cloud.network.lb.LoadBalancingRule.LbStickinessPolicy; +import com.cloud.agent.resource.virtualnetwork.model.LoadBalancerRule.UdpLbEntry; +import com.cloud.utils.Pair; import java.util.List; import java.util.ArrayList; @@ -158,6 +161,65 @@ public void generateConfigurationTestWithSslCert() { Assert.assertTrue(result.contains("bind 10.2.0.1:443 ssl crt /etc/cloudstack/ssl/10_2_0_1-443.pem")); } + private LoadBalancerConfigCommand udpAndTcpCommand(List stickiness) { + final List dests = new ArrayList<>(); + dests.add(new LbDestination(5353, 5353, "10.1.10.2", false)); + dests.add(new LbDestination(5353, 5353, "10.1.10.3", true)); + LoadBalancerTO udp = new LoadBalancerTO("1", "10.2.0.1", 53, "tcp", "source", false, false, false, dests, stickiness); + udp.setLbProtocol("udp"); + udp.setCidrList("1.1.1.1 2.2.2.0/24"); + LoadBalancerTO tcp = new LoadBalancerTO("2", "10.2.0.1", 22, "tcp", "roundrobin", false, false, false, null); + LoadBalancerTO[] lba = new LoadBalancerTO[] {udp, tcp}; + return new LoadBalancerConfigCommand(lba, "10.0.0.1", "10.1.0.1", "10.1.1.1", null, 1L, "12", false, 0L); + } + + @Test + public void generateConfigurationTestUdpRuleNotInHaproxy() { + HAProxyConfigurator hpg = new HAProxyConfigurator(); + LoadBalancerConfigCommand cmd = udpAndTcpCommand(null); + String result = genConfig(hpg, cmd); + Assert.assertTrue(result.contains("listen 10_2_0_1-22")); + Assert.assertFalse(result.contains("10_2_0_1-53")); + + String[][] fwRules = hpg.generateFwRules(cmd); + Assert.assertArrayEquals(new String[] {"10.2.0.1:22:"}, fwRules[LoadBalancerConfigurator.ADD]); + } + + @Test + public void generateUdpEntriesTest() { + HAProxyConfigurator hpg = new HAProxyConfigurator(); + UdpLbEntry[] entries = hpg.generateUdpEntries(udpAndTcpCommand(null)); + Assert.assertEquals(1, entries.length); + Assert.assertEquals("10.2.0.1", entries[0].getSourceIp()); + Assert.assertEquals(53, entries[0].getSourcePort()); + Assert.assertEquals("sh", entries[0].getScheduler()); + Assert.assertEquals(0, entries[0].getPersistence()); + Assert.assertArrayEquals("revoked destinations are left out", new String[] {"10.1.10.2:5353"}, entries[0].getDestinations()); + Assert.assertArrayEquals(new String[] {"1.1.1.1", "2.2.2.0/24"}, entries[0].getCidrs()); + } + + @Test + public void generateUdpEntriesTestSourceStickinessBecomesPersistence() { + List> params = new ArrayList<>(); + params.add(new Pair<>("expire", "30m")); + List stickiness = new ArrayList<>(); + stickiness.add(new LbStickinessPolicy("SourceBased", params)); + UdpLbEntry[] entries = new HAProxyConfigurator().generateUdpEntries(udpAndTcpCommand(stickiness)); + Assert.assertEquals(1800, entries[0].getPersistence()); + + stickiness.clear(); + stickiness.add(new LbStickinessPolicy("SourceBased", new ArrayList<>())); + entries = new HAProxyConfigurator().generateUdpEntries(udpAndTcpCommand(stickiness)); + Assert.assertEquals("default expire is 3h", 10800, entries[0].getPersistence()); + } + + @Test + public void getIpvsSchedulerTest() { + Assert.assertEquals("rr", HAProxyConfigurator.getIpvsScheduler("roundrobin")); + Assert.assertEquals("lc", HAProxyConfigurator.getIpvsScheduler("leastconn")); + Assert.assertEquals("sh", HAProxyConfigurator.getIpvsScheduler("source")); + } + private String genConfig(HAProxyConfigurator hpg, LoadBalancerConfigCommand cmd) { String[] sa = hpg.generateConfiguration(cmd); StringBuilder sb = new StringBuilder(); diff --git a/server/src/main/java/com/cloud/network/lb/LoadBalancingRulesManagerImpl.java b/server/src/main/java/com/cloud/network/lb/LoadBalancingRulesManagerImpl.java index 25c99561b585..466bac3092b3 100644 --- a/server/src/main/java/com/cloud/network/lb/LoadBalancingRulesManagerImpl.java +++ b/server/src/main/java/com/cloud/network/lb/LoadBalancingRulesManagerImpl.java @@ -2327,6 +2327,10 @@ public LoadBalancer updateLoadBalancerRule(UpdateLoadBalancerRuleCmd cmd) { } if (StringUtils.isNotBlank(lbProtocol)) { + if (NetUtils.UDP_PROTO.equals(lbProtocol) != NetUtils.UDP_PROTO.equals(lb.getProtocol())) { + // the rule's firewall protocol (tcp / udp) is fixed when the rule is created + throw new InvalidParameterValueException("The protocol of a load balancer rule cannot be changed between udp and tcp based protocols, create a new rule instead"); + } lb.setLbProtocol(lbProtocol); } diff --git a/server/src/main/java/com/cloud/network/router/NetworkHelperImpl.java b/server/src/main/java/com/cloud/network/router/NetworkHelperImpl.java index 56534a8ee7df..2df2929a8757 100644 --- a/server/src/main/java/com/cloud/network/router/NetworkHelperImpl.java +++ b/server/src/main/java/com/cloud/network/router/NetworkHelperImpl.java @@ -959,10 +959,8 @@ public boolean validateHAProxyLBRule(final LoadBalancingRule rule) { } String lbProtocol = rule.getLbProtocol(); if (lbProtocol != null && lbProtocol.toLowerCase().equals(NetUtils.UDP_PROTO)) { - if (logger.isDebugEnabled()) { - logger.debug("Can't create LB rule as haproxy does not support UDP"); - } - return false; + // UDP rules are not served by haproxy but by IPVS in the router kernel + return validateUdpLBRule(rule); } validateHAproxyLbProtocol(rule.getLbProtocol()); @@ -1020,6 +1018,32 @@ public boolean validateHAProxyLBRule(final LoadBalancingRule rule) { return true; } + private boolean validateUdpLBRule(final LoadBalancingRule rule) { + for (final LoadBalancingRule.LbStickinessPolicy stickinessPolicy : rule.getStickinessPolicies()) { + if (stickinessPolicy.isRevoked()) { + continue; + } + if (!LbStickinessMethod.StickinessMethodType.SourceBased.getName().equalsIgnoreCase(stickinessPolicy.getMethodName())) { + throw new InvalidParameterValueException(String.format("Stickiness method %s is not supported for UDP load balancer rules, only %s", + stickinessPolicy.getMethodName(), LbStickinessMethod.StickinessMethodType.SourceBased.getName())); + } + // "expire" becomes the IPVS persistence timeout + for (final Pair param : stickinessPolicy.getParams()) { + if ("expire".equalsIgnoreCase(param.first()) && param.second() != null && !param.second().trim().matches("\\d+[smhd]?")) { + throw new InvalidParameterValueException(String.format("Invalid expire value %s, expected a number with an optional unit s, m, h or d", param.second())); + } + } + } + if (rule.getHealthCheckPolicies() != null) { + for (final LoadBalancingRule.LbHealthCheckPolicy healthCheckPolicy : rule.getHealthCheckPolicies()) { + if (!healthCheckPolicy.isRevoked()) { + throw new InvalidParameterValueException("Health check policies are not supported for UDP load balancer rules"); + } + } + } + return true; + } + private void validateHAproxyLbProtocol(String lbProtocol) { List lbProtocols = Arrays.asList("tcp", "udp", "tcp-proxy", "ssl"); if (lbProtocol != null && !lbProtocols.contains(lbProtocol)) { diff --git a/server/src/test/java/com/cloud/network/router/NetworkHelperImplTest.java b/server/src/test/java/com/cloud/network/router/NetworkHelperImplTest.java index e4773d01de92..8f8e0ff45c21 100644 --- a/server/src/test/java/com/cloud/network/router/NetworkHelperImplTest.java +++ b/server/src/test/java/com/cloud/network/router/NetworkHelperImplTest.java @@ -69,6 +69,13 @@ import com.cloud.vm.VirtualMachineName; import com.cloud.vm.dao.DomainRouterDao; import com.cloud.vm.dao.NicDao; +import com.cloud.configuration.Config; +import com.cloud.exception.InvalidParameterValueException; +import com.cloud.network.lb.LoadBalancingRule; +import com.cloud.utils.Pair; +import org.apache.cloudstack.framework.config.dao.ConfigurationDao; + +import java.util.ArrayList; import java.util.List; import java.util.Map; @@ -101,6 +108,9 @@ public class NetworkHelperImplTest { @Mock private NicDao nicDaoMock; + @Mock + private ConfigurationDao configDaoMock; + @Mock private RouterDeploymentDefinition routerDeploymentDefinition; @@ -534,4 +544,38 @@ public void configurePublicNicTestUpdateGuestNetworksIpv6Nic() throws Insufficie Map> nic = networkHelperSpy.configurePublicNic(routerDeploymentDefinitionMock, false); Mockito.verify(ipv6ServiceMock).updateNicIpv6(Mockito.eq(nic.get(networkMock).get(0)), Mockito.any(), Mockito.eq(networkMock)); } + + private LoadBalancingRule udpRule(List stickiness) { + when(configDaoMock.getValue(Config.NetworkLBHaproxyStatsPort.key())).thenReturn("8081"); + LoadBalancingRule rule = mock(LoadBalancingRule.class); + when(rule.getSourcePortStart()).thenReturn(53); + when(rule.getLbProtocol()).thenReturn("udp"); + when(rule.getStickinessPolicies()).thenReturn(stickiness); + return rule; + } + + @Test + public void validateHAProxyLBRuleTestUdpAccepted() { + List> params = new ArrayList<>(); + params.add(new Pair<>("expire", "30m")); + List stickiness = new ArrayList<>(); + stickiness.add(new LoadBalancingRule.LbStickinessPolicy("SourceBased", params)); + assertTrue(networkHelperSpy.validateHAProxyLBRule(udpRule(stickiness))); + } + + @Test(expected = InvalidParameterValueException.class) + public void validateHAProxyLBRuleTestUdpRejectsCookieStickiness() { + List stickiness = new ArrayList<>(); + stickiness.add(new LoadBalancingRule.LbStickinessPolicy("LbCookie", new ArrayList<>())); + networkHelperSpy.validateHAProxyLBRule(udpRule(stickiness)); + } + + @Test(expected = InvalidParameterValueException.class) + public void validateHAProxyLBRuleTestUdpRejectsBadExpire() { + List> params = new ArrayList<>(); + params.add(new Pair<>("expire", "soon")); + List stickiness = new ArrayList<>(); + stickiness.add(new LoadBalancingRule.LbStickinessPolicy("SourceBased", params)); + networkHelperSpy.validateHAProxyLBRule(udpRule(stickiness)); + } } diff --git a/systemvm/debian/opt/cloud/bin/cs/CsLoadBalancer.py b/systemvm/debian/opt/cloud/bin/cs/CsLoadBalancer.py index 39de9b55d158..fdde072b1f59 100755 --- a/systemvm/debian/opt/cloud/bin/cs/CsLoadBalancer.py +++ b/systemvm/debian/opt/cloud/bin/cs/CsLoadBalancer.py @@ -28,6 +28,10 @@ SSL_CERTS_DIR = "/etc/cloudstack/ssl/" +# matches "-A -u 1.2.3.4:53 -s rr -p 300" and "-a -u 1.2.3.4:53 -r 10.1.1.5:53 -m -w 1" from ipvsadm-save -n +IPVS_SERVICE_RE = re.compile(r"^-A -u (\S+) -s (\S+)(?: -p (\d+))?") +IPVS_DEST_RE = re.compile(r"^-a -u (\S+) -r (\S+)") + class CsLoadBalancer(CsDataBag): """ Manage Load Balancer entries """ @@ -68,6 +72,9 @@ def process(self): stat_rules = self.dbag['config'][0]['stat_rules'] self._configure_firewall(add_rules, remove_rules, stat_rules) + udp_rules = self.dbag['config'][0].get('udp_rules', []) + self._configure_udp(udp_rules) + def _configure_firewall(self, add_rules, remove_rules, stat_rules): firewall = self.config.get_fw() @@ -93,6 +100,69 @@ def _configure_firewall(self, add_rules, remove_rules, stat_rules): port = path[1] firewall.append(["filter", "", "-A INPUT -p tcp -m tcp -d %s --dport %s -m state --state NEW -j ACCEPT" % (ip, port)]) + def _configure_udp(self, udp_rules): + """ UDP rules are served by IPVS in NAT mode: the router is the gateway of the + backends, so their replies come back through it and get the public address back. """ + firewall = self.config.get_fw() + wanted = {} + for rule in udp_rules: + vip = "%s:%s" % (rule['source_ip'], rule['source_port']) + wanted[vip] = rule + cidrs = [c for c in rule.get('cidrs', []) if c and c != "0.0.0.0/0"] + for cidr in cidrs or [None]: + source = "-s %s " % cidr if cidr else "" + firewall.append(["filter", "", "-A INPUT %s-p udp -m udp -d %s --dport %s -m state --state NEW -j ACCEPT" % + (source, rule['source_ip'], rule['source_port'])]) + + current_services, current_dests = self._get_ipvs_udp() + if not wanted and not current_services: + return + + CsHelper.execute("modprobe ip_vs") + # let netfilter track IPVS connections, so the replies pass the router's stateful rules + CsHelper.execute("sysctl -w net.ipv4.vs.conntrack=1") + CsHelper.execute("sysctl -w net.ipv4.vs.expire_nodest_conn=1") + + for vip in current_services: + if vip not in wanted: + logging.info("CsLoadBalancer:: removing UDP service %s" % vip) + CsHelper.execute("ipvsadm -D -u %s" % vip) + + for vip, rule in wanted.items(): + options = "-s %s" % rule['scheduler'] + if int(rule.get('persistence', 0)) > 0: + options += " -p %s" % rule['persistence'] + if vip not in current_services: + logging.info("CsLoadBalancer:: adding UDP service %s %s" % (vip, options)) + CsHelper.execute("ipvsadm -A -u %s %s" % (vip, options)) + elif current_services[vip] != (rule['scheduler'], int(rule.get('persistence', 0))): + logging.info("CsLoadBalancer:: changing UDP service %s to %s" % (vip, options)) + CsHelper.execute("ipvsadm -E -u %s %s" % (vip, options)) + + dests = set(rule.get('destinations', [])) + have = current_dests.get(vip, set()) + for dest in have - dests: + CsHelper.execute("ipvsadm -d -u %s -r %s" % (vip, dest)) + for dest in dests - have: + CsHelper.execute("ipvsadm -a -u %s -r %s -m" % (vip, dest)) + + def _get_ipvs_udp(self): + """ Returns the UDP services in IPVS as {vip: (scheduler, persistence)} and {vip: set(destinations)} """ + services = {} + dests = {} + if not os.path.exists("/proc/net/ip_vs"): + return services, dests + for line in CsHelper.execute("ipvsadm-save -n"): + m = IPVS_SERVICE_RE.match(line) + if m: + services[m.group(1)] = (m.group(2), int(m.group(3) or 0)) + dests.setdefault(m.group(1), set()) + continue + m = IPVS_DEST_RE.match(line) + if m: + dests.setdefault(m.group(1), set()).add(m.group(2)) + return services, dests + def _create_pem_for_sslcert(self, ssl_certs): logging.debug("CsLoadBalancer:: creating new pem files in %s and cleaning up it" % SSL_CERTS_DIR) if not os.path.exists(SSL_CERTS_DIR):