vr: serve UDP load balancer rules with IPVS

api/src/main/java/org/apache/cloudstack/api/command/user/loadbalancer/CreateLoadBalancerRuleCmd.java

@@ -327,6 +327,10 @@ public Integer getSourcePortEnd() {
     }
 
     public String getProtocol() {
+        // a UDP load balancer needs a UDP firewall rule and must not clash with UDP port forwarding on the same port
+        if (NetUtils.UDP_PROTO.equals(getLbProtocol())) {
+            return NetUtils.UDP_PROTO;
+        }
         return NetUtils.TCP_PROTO;
     }
 

core/src/main/java/com/cloud/agent/resource/virtualnetwork/facade/LoadBalancerConfigItem.java

@@ -58,6 +58,7 @@ public List<ConfigItem> generateConfig(final NetworkElementCommand cmd) {
         final LoadBalancerRule loadBalancerRule = new LoadBalancerRule(configuration, tmpCfgFilePath, tmpCfgFileName, addRules, removeRules, statRules, routerIp);
         final LoadBalancerRule.SslCertEntry[] sslCerts = cfgtr.generateSslCertEntries(command);
         loadBalancerRule.setSslCerts(sslCerts);
+        loadBalancerRule.setUdpRules(cfgtr.generateUdpEntries(command));
 
         final List<LoadBalancerRule> rules = new LinkedList<LoadBalancerRule>();
         rules.add(loadBalancerRule);

core/src/main/java/com/cloud/agent/resource/virtualnetwork/model/LoadBalancerRule.java

@@ -26,6 +26,7 @@ public class LoadBalancerRule {
     private String tmpCfgFilePath;
     private String tmpCfgFileName;
     private SslCertEntry[] sslCerts;
+    private UdpLbEntry[] udpRules;
 
     private String[] addRules;
     private String[] removeRules;
@@ -80,6 +81,50 @@ public String getPassword() {
         }
     }
 
+    /**
+     * A UDP load balancer rule. The router serves it with IPVS (NAT mode), not haproxy.
+     */
+    public static class UdpLbEntry {
+        private String sourceIp;
+        private int sourcePort;
+        // IPVS scheduler: rr, lc or sh
+        private String scheduler;
+        // IPVS persistence timeout in seconds, 0 = none
+        private int persistence;
+        // "ip:port" of each backend
+        private String[] destinations;
+        // allowed source CIDRs, empty = everyone
+        private String[] cidrs;
+
+        public UdpLbEntry(String sourceIp, int sourcePort, String scheduler, int persistence, String[] destinations, String[] cidrs) {
+            this.sourceIp = sourceIp;
+            this.sourcePort = sourcePort;
+            this.scheduler = scheduler;
+            this.persistence = persistence;
+            this.destinations = destinations;
+            this.cidrs = cidrs;
+        }
+
+        public String getSourceIp() {
+            return sourceIp;
+        }
+        public int getSourcePort() {
+            return sourcePort;
+        }
+        public String getScheduler() {
+            return scheduler;
+        }
+        public int getPersistence() {
+            return persistence;
+        }
+        public String[] getDestinations() {
+            return destinations;
+        }
+        public String[] getCidrs() {
+            return cidrs;
+        }
+    }
+
     public LoadBalancerRule() {
         // Empty constructor for (de)serialization
     }
@@ -157,4 +202,12 @@ public SslCertEntry[] getSslCerts() {
     public void setSslCerts(final SslCertEntry[] sslCerts) {
         this.sslCerts = sslCerts;
     }
+
+    public UdpLbEntry[] getUdpRules() {
+        return udpRules;
+    }
+
+    public void setUdpRules(final UdpLbEntry[] udpRules) {
+        this.udpRules = udpRules;
+    }
 }

core/src/main/java/com/cloud/network/HAProxyConfigurator.java

@@ -37,6 +37,7 @@
 import com.cloud.agent.api.to.LoadBalancerTO.StickinessPolicyTO;
 import com.cloud.agent.api.to.PortForwardingRuleTO;
 import com.cloud.agent.resource.virtualnetwork.model.LoadBalancerRule.SslCertEntry;
+import com.cloud.agent.resource.virtualnetwork.model.LoadBalancerRule.UdpLbEntry;
 import com.cloud.network.lb.LoadBalancingRule.LbSslCert;
 import com.cloud.network.rules.LbStickinessMethod.StickinessMethodType;
 import com.cloud.utils.Pair;
@@ -683,7 +684,7 @@ public String[] generateConfiguration(final LoadBalancerConfigCommand lbCmd) {
         result.add(blankLine);
         boolean has_listener = false;
         for (final LoadBalancerTO lbTO : lbCmd.getLoadBalancers()) {
-            if (lbTO.isRevoked()) {
+            if (lbTO.isRevoked() || isUdp(lbTO)) {
                 continue;
             }
             final List<String> poolRules = getRulesForPool(lbTO, lbCmd);
@@ -708,6 +709,9 @@ public String[][] generateFwRules(final LoadBalancerConfigCommand lbCmd) {
         final Set<String> toStats = new HashSet<String>();
 
         for (final LoadBalancerTO lbTO : lbCmd.getLoadBalancers()) {
+            if (isUdp(lbTO)) {
+                continue;
+            }
 
             final StringBuilder sb = new StringBuilder();
             sb.append(lbTO.getSrcIp()).append(":");
@@ -764,4 +768,73 @@ private void addSslCertEntry(Set<SslCertEntry> sslCertEntries, LoadBalancerTO lb
         final SslCertEntry sslCertEntry = new SslCertEntry(name, cert.getCert(), cert.getKey(), cert.getChain(), cert.getPassword());
         sslCertEntries.add(sslCertEntry);
     }
+
+    private static boolean isUdp(final LoadBalancerTO lbTO) {
+        return NetUtils.UDP_PROTO.equalsIgnoreCase(lbTO.getLbProtocol());
+    }
+
+    @Override
+    public UdpLbEntry[] generateUdpEntries(final LoadBalancerConfigCommand lbCmd) {
+        final List<UdpLbEntry> result = new ArrayList<>();
+        for (final LoadBalancerTO lbTO : lbCmd.getLoadBalancers()) {
+            if (lbTO.isRevoked() || !isUdp(lbTO)) {
+                continue;
+            }
+            final List<String> destinations = new ArrayList<>();
+            for (final DestinationTO dest : lbTO.getDestinations()) {
+                if (!dest.isRevoked()) {
+                    destinations.add(dest.getDestIp() + ":" + dest.getDestPort());
+                }
+            }
+            final String[] cidrs = StringUtils.isBlank(lbTO.getCidrList()) ? new String[0] : lbTO.getCidrList().trim().split("\\s+");
+            result.add(new UdpLbEntry(lbTO.getSrcIp(), lbTO.getSrcPort(), getIpvsScheduler(lbTO.getAlgorithm()), getIpvsPersistence(lbTO),
+                    destinations.toArray(new String[0]), cidrs));
+        }
+        return result.toArray(new UdpLbEntry[0]);
+    }
+
+    protected static String getIpvsScheduler(final String algorithm) {
+        if ("leastconn".equals(algorithm)) {
+            return "lc";
+        }
+        if ("source".equals(algorithm)) {
+            return "sh";
+        }
+        return "rr";
+    }
+
+    /**
+     * Source based stickiness becomes IPVS persistence; its "expire" value (default 3h, as for haproxy) is the timeout.
+     */
+    protected static int getIpvsPersistence(final LoadBalancerTO lbTO) {
+        if (lbTO.getStickinessPolicies() == null) {
+            return 0;
+        }
+        for (final StickinessPolicyTO policy : lbTO.getStickinessPolicies()) {
+            if (policy == null || !StickinessMethodType.SourceBased.getName().equalsIgnoreCase(policy.getMethodName())) {
+                continue;
+            }
+            String expire = "3h";
+            for (final Pair<String, String> param : policy.getParams()) {
+                if ("expire".equalsIgnoreCase(param.first()) && StringUtils.isNotBlank(param.second())) {
+                    expire = param.second().trim();
+                }
+            }
+            return durationToSeconds(expire);
+        }
+        return 0;
+    }
+
+    private static int durationToSeconds(final String duration) {
+        final char unit = duration.charAt(duration.length() - 1);
+        final int multiplier;
+        switch (unit) {
+            case 's': multiplier = 1; break;
+            case 'm': multiplier = 60; break;
+            case 'h': multiplier = 3600; break;
+            case 'd': multiplier = 86400; break;
+            default: return Integer.parseInt(duration);
+        }
+        return Integer.parseInt(duration.substring(0, duration.length() - 1)) * multiplier;
+    }
 }

core/src/main/java/com/cloud/network/LoadBalancerConfigurator.java

@@ -24,6 +24,7 @@
 import com.cloud.agent.api.routing.LoadBalancerConfigCommand;
 import com.cloud.agent.api.to.PortForwardingRuleTO;
 import com.cloud.agent.resource.virtualnetwork.model.LoadBalancerRule.SslCertEntry;
+import com.cloud.agent.resource.virtualnetwork.model.LoadBalancerRule.UdpLbEntry;
 
 public interface LoadBalancerConfigurator {
     public final static int ADD = 0;
@@ -37,4 +38,6 @@ public interface LoadBalancerConfigurator {
     public String[][] generateFwRules(LoadBalancerConfigCommand lbCmd);
 
     public SslCertEntry[] generateSslCertEntries(LoadBalancerConfigCommand lbCmd);
+
+    public UdpLbEntry[] generateUdpEntries(LoadBalancerConfigCommand lbCmd);
 }

core/src/test/java/com/cloud/network/HAProxyConfiguratorTest.java

@@ -32,6 +32,9 @@
 import com.cloud.agent.api.to.LoadBalancerTO;
 import com.cloud.network.lb.LoadBalancingRule.LbDestination;
 import com.cloud.network.lb.LoadBalancingRule.LbSslCert;
+import com.cloud.network.lb.LoadBalancingRule.LbStickinessPolicy;
+import com.cloud.agent.resource.virtualnetwork.model.LoadBalancerRule.UdpLbEntry;
+import com.cloud.utils.Pair;
 
 import java.util.List;
 import java.util.ArrayList;
@@ -158,6 +161,65 @@ public void generateConfigurationTestWithSslCert() {
         Assert.assertTrue(result.contains("bind 10.2.0.1:443 ssl crt /etc/cloudstack/ssl/10_2_0_1-443.pem"));
     }
 
+    private LoadBalancerConfigCommand udpAndTcpCommand(List<LbStickinessPolicy> stickiness) {
+        final List<LbDestination> dests = new ArrayList<>();
+        dests.add(new LbDestination(5353, 5353, "10.1.10.2", false));
+        dests.add(new LbDestination(5353, 5353, "10.1.10.3", true));
+        LoadBalancerTO udp = new LoadBalancerTO("1", "10.2.0.1", 53, "tcp", "source", false, false, false, dests, stickiness);
+        udp.setLbProtocol("udp");
+        udp.setCidrList("1.1.1.1 2.2.2.0/24");
+        LoadBalancerTO tcp = new LoadBalancerTO("2", "10.2.0.1", 22, "tcp", "roundrobin", false, false, false, null);
+        LoadBalancerTO[] lba = new LoadBalancerTO[] {udp, tcp};
+        return new LoadBalancerConfigCommand(lba, "10.0.0.1", "10.1.0.1", "10.1.1.1", null, 1L, "12", false, 0L);
+    }
+
+    @Test
+    public void generateConfigurationTestUdpRuleNotInHaproxy() {
+        HAProxyConfigurator hpg = new HAProxyConfigurator();
+        LoadBalancerConfigCommand cmd = udpAndTcpCommand(null);
+        String result = genConfig(hpg, cmd);
+        Assert.assertTrue(result.contains("listen 10_2_0_1-22"));
+        Assert.assertFalse(result.contains("10_2_0_1-53"));
+
+        String[][] fwRules = hpg.generateFwRules(cmd);
+        Assert.assertArrayEquals(new String[] {"10.2.0.1:22:"}, fwRules[LoadBalancerConfigurator.ADD]);
+    }
+
+    @Test
+    public void generateUdpEntriesTest() {
+        HAProxyConfigurator hpg = new HAProxyConfigurator();
+        UdpLbEntry[] entries = hpg.generateUdpEntries(udpAndTcpCommand(null));
+        Assert.assertEquals(1, entries.length);
+        Assert.assertEquals("10.2.0.1", entries[0].getSourceIp());
+        Assert.assertEquals(53, entries[0].getSourcePort());
+        Assert.assertEquals("sh", entries[0].getScheduler());
+        Assert.assertEquals(0, entries[0].getPersistence());
+        Assert.assertArrayEquals("revoked destinations are left out", new String[] {"10.1.10.2:5353"}, entries[0].getDestinations());
+        Assert.assertArrayEquals(new String[] {"1.1.1.1", "2.2.2.0/24"}, entries[0].getCidrs());
+    }
+
+    @Test
+    public void generateUdpEntriesTestSourceStickinessBecomesPersistence() {
+        List<Pair<String, String>> params = new ArrayList<>();
+        params.add(new Pair<>("expire", "30m"));
+        List<LbStickinessPolicy> stickiness = new ArrayList<>();
+        stickiness.add(new LbStickinessPolicy("SourceBased", params));
+        UdpLbEntry[] entries = new HAProxyConfigurator().generateUdpEntries(udpAndTcpCommand(stickiness));
+        Assert.assertEquals(1800, entries[0].getPersistence());
+
+        stickiness.clear();
+        stickiness.add(new LbStickinessPolicy("SourceBased", new ArrayList<>()));
+        entries = new HAProxyConfigurator().generateUdpEntries(udpAndTcpCommand(stickiness));
+        Assert.assertEquals("default expire is 3h", 10800, entries[0].getPersistence());
+    }
+
+    @Test
+    public void getIpvsSchedulerTest() {
+        Assert.assertEquals("rr", HAProxyConfigurator.getIpvsScheduler("roundrobin"));
+        Assert.assertEquals("lc", HAProxyConfigurator.getIpvsScheduler("leastconn"));
+        Assert.assertEquals("sh", HAProxyConfigurator.getIpvsScheduler("source"));
+    }
+
     private String genConfig(HAProxyConfigurator hpg, LoadBalancerConfigCommand cmd) {
         String[] sa = hpg.generateConfiguration(cmd);
         StringBuilder sb = new StringBuilder();

server/src/main/java/com/cloud/network/lb/LoadBalancingRulesManagerImpl.java

@@ -2327,6 +2327,10 @@ public LoadBalancer updateLoadBalancerRule(UpdateLoadBalancerRuleCmd cmd) {
         }
 
         if (StringUtils.isNotBlank(lbProtocol)) {
+            if (NetUtils.UDP_PROTO.equals(lbProtocol) != NetUtils.UDP_PROTO.equals(lb.getProtocol())) {
+                // the rule's firewall protocol (tcp / udp) is fixed when the rule is created
+                throw new InvalidParameterValueException("The protocol of a load balancer rule cannot be changed between udp and tcp based protocols, create a new rule instead");
+            }
             lb.setLbProtocol(lbProtocol);
         }
 

server/src/main/java/com/cloud/network/router/NetworkHelperImpl.java

@@ -959,10 +959,8 @@ public boolean validateHAProxyLBRule(final LoadBalancingRule rule) {
         }
         String lbProtocol = rule.getLbProtocol();
         if (lbProtocol != null && lbProtocol.toLowerCase().equals(NetUtils.UDP_PROTO)) {
-            if (logger.isDebugEnabled()) {
-                logger.debug("Can't create LB rule as haproxy does not support UDP");
-            }
-            return false;
+            // UDP rules are not served by haproxy but by IPVS in the router kernel
+            return validateUdpLBRule(rule);
         }
 
         validateHAproxyLbProtocol(rule.getLbProtocol());
@@ -1020,6 +1018,32 @@ public boolean validateHAProxyLBRule(final LoadBalancingRule rule) {
         return true;
     }
 
+    private boolean validateUdpLBRule(final LoadBalancingRule rule) {
+        for (final LoadBalancingRule.LbStickinessPolicy stickinessPolicy : rule.getStickinessPolicies()) {
+            if (stickinessPolicy.isRevoked()) {
+                continue;
+            }
+            if (!LbStickinessMethod.StickinessMethodType.SourceBased.getName().equalsIgnoreCase(stickinessPolicy.getMethodName())) {
+                throw new InvalidParameterValueException(String.format("Stickiness method %s is not supported for UDP load balancer rules, only %s",
+                        stickinessPolicy.getMethodName(), LbStickinessMethod.StickinessMethodType.SourceBased.getName()));
+            }
+            // "expire" becomes the IPVS persistence timeout
+            for (final Pair<String, String> param : stickinessPolicy.getParams()) {
+                if ("expire".equalsIgnoreCase(param.first()) && param.second() != null && !param.second().trim().matches("\\d+[smhd]?")) {
+                    throw new InvalidParameterValueException(String.format("Invalid expire value %s, expected a number with an optional unit s, m, h or d", param.second()));
+                }
+            }
+        }
+        if (rule.getHealthCheckPolicies() != null) {
+            for (final LoadBalancingRule.LbHealthCheckPolicy healthCheckPolicy : rule.getHealthCheckPolicies()) {
+                if (!healthCheckPolicy.isRevoked()) {
+                    throw new InvalidParameterValueException("Health check policies are not supported for UDP load balancer rules");
+                }
+            }
+        }
+        return true;
+    }
+
     private void validateHAproxyLbProtocol(String lbProtocol) {
         List<String> lbProtocols = Arrays.asList("tcp", "udp", "tcp-proxy", "ssl");
         if (lbProtocol != null && !lbProtocols.contains(lbProtocol)) {