diff --git a/engine/schema/pom.xml b/engine/schema/pom.xml index 6e078d7ec9f8..a92845184e95 100644 --- a/engine/schema/pom.xml +++ b/engine/schema/pom.xml @@ -88,8 +88,15 @@ diff --git a/engine/schema/templateConfig.sh b/engine/schema/templateConfig.sh index 7a8283e77c1c..71b395689c15 100755 --- a/engine/schema/templateConfig.sh +++ b/engine/schema/templateConfig.sh @@ -28,9 +28,7 @@ function getTemplateVersion() { subversion2="$(cut -d'.' -f2 <<<"$version")" minorversion="$(cut -d'.' -f3 <<<"$version")" fourthversion="$(cut -d'.' -f4 <<<"$version")" - export CS_VERSION="${subversion1}"."${subversion2}" export CS_MINOR_VERSION="${minorversion}" - export VERSION="${CS_VERSION}.${CS_MINOR_VERSION}" if [[ "$subversion1" -ge "$NEW_VERSIONING_CUTOVER_MAJOR_VERSION" ]]; then # New versioning (major.minor.security): the third component is the # security release itself, there is no separate patch component. A @@ -40,11 +38,17 @@ function getTemplateVersion() { echo "Invalid version '${projVersion}': major versions at or above ${NEW_VERSIONING_CUTOVER_MAJOR_VERSION} do not support a 4-component major.minor.patch.security format" >&2 exit 1 fi - export FULL_VERSION="${VERSION}" + # The templates are in the directory named after the major version (24), + # and the file names contain major.minor (24.0), like 4.22 and 4.22.0 before + export CS_VERSION="${subversion1}" + export VERSION="${subversion1}.${subversion2}" + export FULL_VERSION="${subversion1}.${subversion2}.${minorversion}" else # Legacy versioning: major.minor.patch, or major.minor.patch.security when a 4th # component is present. Without a 4th component, VERSION is already correct as-is; # appending an empty security component would leave a dangling trailing dot. + export CS_VERSION="${subversion1}"."${subversion2}" + export VERSION="${CS_VERSION}.${CS_MINOR_VERSION}" if [[ -n "$fourthversion" ]]; then export FULL_VERSION="${VERSION}.${fourthversion}" else diff --git a/engine/schema/test_templateConfig.sh b/engine/schema/test_templateConfig.sh index 2acb4281edb7..95cab07fb280 100755 --- a/engine/schema/test_templateConfig.sh +++ b/engine/schema/test_templateConfig.sh @@ -51,6 +51,7 @@ assert_eq "legacy: VERSION" "4.22.0" "$VERSION" assert_eq "legacy: FULL_VERSION keeps the 4th (security) component" "4.22.0.0" "$FULL_VERSION" getTemplateVersion "4.23.0.1-SNAPSHOT" +assert_eq "legacy with -SNAPSHOT suffix: CS_VERSION" "4.23" "$CS_VERSION" assert_eq "legacy with -SNAPSHOT suffix: VERSION" "4.23.0" "$VERSION" assert_eq "legacy with -SNAPSHOT suffix: FULL_VERSION" "4.23.0.1" "$FULL_VERSION" @@ -62,16 +63,18 @@ assert_eq "legacy 3-component (no security): FULL_VERSION has no trailing dot" " # --- getTemplateVersion: new versioning (major >= 24, post-cutover) --- getTemplateVersion "24.0.0" -assert_eq "cutover: CS_VERSION" "24.0" "$CS_VERSION" -assert_eq "cutover: VERSION" "24.0.0" "$VERSION" +assert_eq "cutover: CS_VERSION (directory) is the major version" "24" "$CS_VERSION" +assert_eq "cutover: VERSION (file names) is major.minor" "24.0" "$VERSION" assert_eq "cutover: FULL_VERSION has no trailing dot (patch dropped)" "24.0.0" "$FULL_VERSION" getTemplateVersion "24.1.2-SNAPSHOT" -assert_eq "cutover with -SNAPSHOT suffix: VERSION" "24.1.2" "$VERSION" +assert_eq "cutover with -SNAPSHOT suffix: CS_VERSION" "24" "$CS_VERSION" +assert_eq "cutover with -SNAPSHOT suffix: VERSION" "24.1" "$VERSION" assert_eq "cutover with -SNAPSHOT suffix: FULL_VERSION" "24.1.2" "$FULL_VERSION" getTemplateVersion "99.9.9" -assert_eq "future major: VERSION" "99.9.9" "$VERSION" +assert_eq "future major: CS_VERSION" "99" "$CS_VERSION" +assert_eq "future major: VERSION" "99.9" "$VERSION" assert_eq "future major: FULL_VERSION" "99.9.9" "$FULL_VERSION" # A 4-component major.minor.patch.security value is invalid at/after the cutover (matches @@ -96,7 +99,7 @@ run_create_metadata_file() { : > "$METADATAFILE" createMetadataFile - grep '^version = ' "$METADATAFILE" | head -1 | cut -d' ' -f3 + grep "^${2:-version} = " "$METADATAFILE" | head -1 | cut -d' ' -f3 rm -rf "$workdir" } @@ -107,6 +110,26 @@ assert_eq "metadata.ini legacy 'version' line" "4.22.0.0" "$actual" actual="$(run_create_metadata_file "24.0.0")" assert_eq "metadata.ini cutover 'version' line has no trailing dot" "24.0.0" "$actual" +# --- createMetadataFile: file names, directory, template name and checksum --- + +actual="$(run_create_metadata_file "4.22.0.0" downloadurl)" +assert_eq "metadata.ini legacy downloadurl" "https://download.cloudstack.org/systemvm//4.22/systemvmtemplate-4.22.0-x86_64-kvm.qcow2.bz2" "$actual" +actual="$(run_create_metadata_file "4.22.0.0" templatename)" +assert_eq "metadata.ini legacy templatename" "systemvm-kvm-4.22.0-x86_64" "$actual" +actual="$(run_create_metadata_file "4.22.0.0" checksum)" +assert_eq "metadata.ini legacy checksum is found" "abc123" "$actual" + +actual="$(run_create_metadata_file "24.0.0" downloadurl)" +assert_eq "metadata.ini cutover downloadurl" "https://download.cloudstack.org/systemvm//24/systemvmtemplate-24.0-x86_64-kvm.qcow2.bz2" "$actual" +actual="$(run_create_metadata_file "24.0.0" filename)" +assert_eq "metadata.ini cutover filename" "systemvmtemplate-24.0-x86_64-kvm.qcow2.bz2" "$actual" +actual="$(run_create_metadata_file "24.0.0" templatename)" +assert_eq "metadata.ini cutover templatename" "systemvm-kvm-24.0-x86_64" "$actual" +actual="$(run_create_metadata_file "24.0.0" checksum)" +assert_eq "metadata.ini cutover checksum is found" "abc123" "$actual" +actual="$(run_create_metadata_file "24.0.1" downloadurl)" +assert_eq "metadata.ini cutover security release uses the same template" "https://download.cloudstack.org/systemvm//24/systemvmtemplate-24.0-x86_64-kvm.qcow2.bz2" "$actual" + echo "" echo "${pass} passed, ${fail} failed" [[ $fail -eq 0 ]] diff --git a/pom.xml b/pom.xml index 80111b26a4ab..8a1c7a27652d 100644 --- a/pom.xml +++ b/pom.xml @@ -50,7 +50,7 @@ UTF-8 UTF-8 https://download.cloudstack.org/systemvm - 4.22.0.0 + 24.0.0 apache https://sonarcloud.io engine/schema/src/main/java/org/apache/cloudstack/backup/BackupOfferingDetailsVO.java diff --git a/scripts/installer/export-templates.sh b/scripts/installer/export-templates.sh index 820596837131..5d38ac54472a 100755 --- a/scripts/installer/export-templates.sh +++ b/scripts/installer/export-templates.sh @@ -26,8 +26,8 @@ TEMPLATE_VERSION=$(awk -F "=" '/version/ {print $2}' ${IMAGE_PATH}${METADATA_FIL TEMPLATE_PATH="/usr/share/cloudstack-management/templates/systemvm/" TEMPLATE_MAJOR_VERSION="$(cut -d'.' -f1 <<<"$TEMPLATE_VERSION")" if [[ "$TEMPLATE_MAJOR_VERSION" -ge "$NEW_VERSIONING_CUTOVER_MAJOR_VERSION" ]]; then - # New versioning (major.minor.security): the generated template files use the full version as-is. - VERSION="${TEMPLATE_VERSION}" + # New versioning (major.minor.security): the template files contain major.minor, without the security component. + VERSION="$(cut -d'.' -f1,2 <<<"$TEMPLATE_VERSION")" else # Legacy versioning drops the trailing security component only for four-component metadata; # a valid three-component major.minor.patch value is already the filename version. diff --git a/systemvm/debian/opt/cloud/bin/passwd_server_ip.py b/systemvm/debian/opt/cloud/bin/passwd_server_ip.py index bf1eab2db02f..5d1ac4c34f12 100755 --- a/systemvm/debian/opt/cloud/bin/passwd_server_ip.py +++ b/systemvm/debian/opt/cloud/bin/passwd_server_ip.py @@ -26,7 +26,7 @@ # curl --header 'DomU_Request: save_password' http://localhost:8080/ -F ip= -F password= import binascii -import cgi +import email.policy import os import socketserver import sys @@ -34,6 +34,7 @@ import threading import urllib.parse +from email.parser import BytesParser from http.server import BaseHTTPRequestHandler, HTTPServer from socketserver import ThreadingMixIn @@ -44,6 +45,26 @@ allowAddresses = ['localhost', '127.0.0.1'] lock = threading.RLock() + +def parseForm(contentType, body): + """ Parse a POST body (multipart/form-data, as sent by curl -F, or + application/x-www-form-urlencoded) to a dict of field name -> value. + The cgi module, which was used before, is not available in Python 3.13 """ + form = {} + contentType = contentType or '' + if contentType.lower().startswith('multipart/form-data'): + message = BytesParser(policy=email.policy.HTTP).parsebytes( + b'Content-Type: ' + contentType.encode() + b'\r\n\r\n' + body) + if message.is_multipart(): + for part in message.iter_parts(): + name = part.get_param('name', header='content-disposition') + if name: + form[name] = part.get_payload(decode=True).decode('utf-8', 'replace') + else: + for name, values in urllib.parse.parse_qs(body.decode('utf-8', 'replace'), keep_blank_values=True).items(): + form[name] = values[0] + return form + def getTokenFile(): return '/tmp/passwdsrvrtoken' @@ -140,12 +161,8 @@ def do_GET(self): return def do_POST(self): - form = cgi.FieldStorage( - fp=self.rfile, - headers=self.headers, - environ={'REQUEST_METHOD':'POST', - 'CONTENT_TYPE':self.headers['content-type'], - }) + form = parseForm(self.headers.get('content-type'), + self.rfile.read(int(self.headers.get('content-length', 0)))) self.send_response(200) self.end_headers() clientAddress = self.client_address[0] @@ -157,13 +174,13 @@ def do_POST(self): syslog.syslog('serve_password: request trying to save password does not contain both ip and password') self.send_response(403) return - token = form['token'].value + token = form['token'] if not checkToken(token): syslog.syslog('serve_password: invalid save_password token received from %s' % clientAddress) self.send_response(403) return - ip = form['ip'].value - password = form['password'].value + ip = form['ip'] + password = form['password'] if not ip or not password: syslog.syslog('serve_password: empty ip/password[%s/%s] received from savepassword' % (ip, password)) return diff --git a/systemvm/debian/opt/cloud/bin/setup/bootstrap.sh b/systemvm/debian/opt/cloud/bin/setup/bootstrap.sh index c601f6ad2210..1e1773f9ec22 100755 --- a/systemvm/debian/opt/cloud/bin/setup/bootstrap.sh +++ b/systemvm/debian/opt/cloud/bin/setup/bootstrap.sh @@ -104,12 +104,29 @@ config_sysctl() { sysctl -p } +# Debian 13 and later does not install /etc/iproute2/rt_tables, which the scripts of all the +# systemvm types use to add the route tables +config_rt_tables() { + if [ ! -f /etc/iproute2/rt_tables ]; then + log_it "Creating /etc/iproute2/rt_tables" + mkdir -p /etc/iproute2 + if [ -f /etc/iptables/rt_tables_init ]; then + cp -f /etc/iptables/rt_tables_init /etc/iproute2/rt_tables + elif [ -f /usr/share/iproute2/rt_tables ]; then + cp -f /usr/share/iproute2/rt_tables /etc/iproute2/rt_tables + else + touch /etc/iproute2/rt_tables + fi + fi +} + bootstrap() { log_it "Bootstrapping systemvm appliance" export TYPE=$(grep -Po 'type=\K[a-zA-Z]*' $CMDLINE) patch config_sysctl + config_rt_tables log_it "Configuring systemvm type=$TYPE" if [ -f "/opt/cloud/bin/setup/$TYPE.sh" ]; then diff --git a/systemvm/debian/opt/cloud/bin/setup/cksnode.sh b/systemvm/debian/opt/cloud/bin/setup/cksnode.sh index e1541f6d0bd1..39d9c65a6371 100755 --- a/systemvm/debian/opt/cloud/bin/setup/cksnode.sh +++ b/systemvm/debian/opt/cloud/bin/setup/cksnode.sh @@ -68,12 +68,9 @@ setup_k8s_node() { log_it "Starting cloud-init services" systemctl enable --now --no-block containerd if [ -f /home/cloud/success ]; then - systemctl stop cloud-init cloud-config cloud-final - systemctl disable cloud-init cloud-config cloud-final + stop_cloud_init else - systemctl start --no-block cloud-init - systemctl start --no-block cloud-config - systemctl start --no-block cloud-final + start_cloud_init fi } diff --git a/systemvm/debian/opt/cloud/bin/setup/common.sh b/systemvm/debian/opt/cloud/bin/setup/common.sh index ef1576ab588c..012a2a336c2c 100755 --- a/systemvm/debian/opt/cloud/bin/setup/common.sh +++ b/systemvm/debian/opt/cloud/bin/setup/common.sh @@ -700,6 +700,34 @@ setup_ntp() { fi } +# cloud-init is disabled in the template with /etc/cloud/cloud-init.disabled, and the units of +# cloud-init 25.1 and later (Debian 13) are skipped by systemd while that file exists. The generator +# of cloud-init decides at boot, so it has to run again after the file is removed. +# cloud-init 25.1 and later also does not have cloud-init.service anymore: cloud-init-main is +# a single process which runs all the stages, and the units of the stages hand over to it +start_cloud_init() { + rm -f /etc/cloud/cloud-init.disabled /run/cloud-init/disabled + systemctl daemon-reload + if systemctl cat cloud-init-main.service > /dev/null 2>&1; then + systemctl start --no-block cloud-init-main cloud-init-local cloud-init-network cloud-config cloud-final + else + systemctl start --no-block cloud-init + systemctl start --no-block cloud-config + systemctl start --no-block cloud-final + fi +} + +stop_cloud_init() { + local svc + for svc in cloud-init cloud-init-main cloud-init-local cloud-init-network cloud-config cloud-final; do + if systemctl cat ${svc}.service > /dev/null 2>&1; then + systemctl stop ${svc} + systemctl disable ${svc} + fi + done + touch /etc/cloud/cloud-init.disabled +} + routing_svcs() { echo "haproxy apache2 frr" > /var/cache/cloud/enabled_svcs echo "cloud nfs-common portmap" > /var/cache/cloud/disabled_svcs diff --git a/systemvm/debian/opt/cloud/bin/setup/default.sh b/systemvm/debian/opt/cloud/bin/setup/default.sh index 4272e645d416..36292fa05345 100755 --- a/systemvm/debian/opt/cloud/bin/setup/default.sh +++ b/systemvm/debian/opt/cloud/bin/setup/default.sh @@ -23,7 +23,6 @@ setup_default() { auto lo iface lo inet loopback EOF - cp -f /etc/iptables/rt_tables_init /etc/iproute2/rt_tables } setup_default diff --git a/systemvm/debian/opt/cloud/bin/setup/sharedfsvm.sh b/systemvm/debian/opt/cloud/bin/setup/sharedfsvm.sh index 1b1bb2737534..ca6d2d106940 100755 --- a/systemvm/debian/opt/cloud/bin/setup/sharedfsvm.sh +++ b/systemvm/debian/opt/cloud/bin/setup/sharedfsvm.sh @@ -57,12 +57,9 @@ setup_sharedfsvm() { log_it "Starting cloud-init services" if [ -f /home/cloud/success ]; then - systemctl stop cloud-init cloud-config cloud-final - systemctl disable cloud-init cloud-config cloud-final + stop_cloud_init else - systemctl start --no-block cloud-init - systemctl start --no-block cloud-config - systemctl start --no-block cloud-final + start_cloud_init fi } diff --git a/systemvm/test/TestPasswdServerIp.py b/systemvm/test/TestPasswdServerIp.py new file mode 100644 index 000000000000..2774ae037bbd --- /dev/null +++ b/systemvm/test/TestPasswdServerIp.py @@ -0,0 +1,84 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +import shutil +import subprocess +import threading +import unittest +from http.server import BaseHTTPRequestHandler, HTTPServer + +import passwd_server_ip + + +class TestPasswdServerIp(unittest.TestCase): + + def multipart(self, fields, boundary="----boundary123"): + body = b"" + for name, value in fields.items(): + body += ("--%s\r\nContent-Disposition: form-data; name=\"%s\"\r\n\r\n" % (boundary, name)).encode() + body += value.encode() + b"\r\n" + body += ("--%s--\r\n" % boundary).encode() + return "multipart/form-data; boundary=%s" % boundary, body + + def test_parse_multipart_form(self): + contentType, body = self.multipart({"ip": "10.1.1.5", "password": "Ab1&=;'\"x y", "token": "abc"}) + form = passwd_server_ip.parseForm(contentType, body) + self.assertEqual({"ip": "10.1.1.5", "password": "Ab1&=;'\"x y", "token": "abc"}, form) + + def test_parse_multipart_form_with_non_ascii_value(self): + contentType, body = self.multipart({"password": "päss"}) + self.assertEqual("päss", passwd_server_ip.parseForm(contentType, body)["password"]) + + def test_parse_urlencoded_form(self): + form = passwd_server_ip.parseForm("application/x-www-form-urlencoded", b"ip=10.1.1.5&password=a%26b%3Dc&token=abc") + self.assertEqual({"ip": "10.1.1.5", "password": "a&b=c", "token": "abc"}, form) + + def test_parse_form_without_fields(self): + self.assertEqual({}, passwd_server_ip.parseForm(None, b"")) + contentType, body = self.multipart({}) + self.assertEqual({}, passwd_server_ip.parseForm(contentType, body)) + + @unittest.skipUnless(shutil.which("curl"), "curl is not available") + def test_parse_form_posted_by_curl(self): + """ the same request as /opt/cloud/bin/savepassword.sh """ + received = {} + + class Handler(BaseHTTPRequestHandler): + def do_POST(self): + length = int(self.headers.get('content-length', 0)) + received.update(passwd_server_ip.parseForm(self.headers.get('content-type'), self.rfile.read(length))) + self.send_response(200) + self.end_headers() + + def log_message(self, format, *args): + return + + server = HTTPServer(("127.0.0.1", 0), Handler) + thread = threading.Thread(target=server.serve_forever) + thread.start() + try: + subprocess.check_call(["curl", "-s", "--header", "DomU_Request: save_password", + "http://127.0.0.1:%d/" % server.server_address[1], + "-F", "ip=10.1.1.5", "-F", "password=Pa$$w&rd=1", "-F", "token=abc"]) + finally: + server.shutdown() + thread.join() + server.server_close() + self.assertEqual({"ip": "10.1.1.5", "password": "Pa$$w&rd=1", "token": "abc"}, received) + + +if __name__ == '__main__': + unittest.main() diff --git a/test/integration/smoke/test_network_ipv6.py b/test/integration/smoke/test_network_ipv6.py index 1e5cec7ef4c2..3b715c1bcf96 100644 --- a/test/integration/smoke/test_network_ipv6.py +++ b/test/integration/smoke/test_network_ipv6.py @@ -657,8 +657,15 @@ def checkIpv6NetworkRouting(self): cmd.id = fw2.id self.userapiclient.deleteIpv6FirewallRule(cmd) + def isIpv6FirewallRuleListed(self, rules, nft_output): + """ rules are the accepted representations of the rule in the nft output, with or without the braces """ + for rule in rules: + if rule in nft_output or rule.replace("{ ", "").replace(" }", "") in nft_output: + return True + return False + def createAndVerifyIpv6FirewallRule(self, traffic_type, source_cidr, dest_cidr, protocol, - start_port, end_port, icmp_type, icmp_code, parsed_rule, delete=False): + start_port, end_port, icmp_type, icmp_code, parsed_rule, delete=False, alternative_rules=None): self.logger.debug("createAndVerifyIpv6FirewallRule - %s" % parsed_rule) fw_rule = self.createIpv6FirewallRuleInNetwork(self.network.id, traffic_type, source_cidr, dest_cidr, protocol, start_port, end_port, icmp_type, icmp_code) @@ -697,16 +704,16 @@ def createAndVerifyIpv6FirewallRule(self, traffic_type, source_cidr, dest_cidr, "IPv6 firewall rule ICMP code mismatch %d, %d" % (rule.icmpcode, icmp_code)) routerCmd = "nft list chain ip6 %s %s" % (FIREWALL_TABLE, FIREWALL_CHAINS[traffic_type]) res = self.getRouterProcessStatus(self.getNetworkRouter(self.network), routerCmd) - parsed_rule_new = parsed_rule.replace("{ ", "").replace(" }", "") - self.assertTrue(parsed_rule in res or parsed_rule_new in res, - "Listing firewall rule with nft list chain failure for rule: '%s' is not in '%s'" % (parsed_rule, res)) + accepted_rules = [parsed_rule] + (alternative_rules or []) + self.assertTrue(self.isIpv6FirewallRuleListed(accepted_rules, res), + "Listing firewall rule with nft list chain failure for rule: '%s' is not in '%s'" % (accepted_rules, res)) if delete == True: cmd = deleteIpv6FirewallRule.deleteIpv6FirewallRuleCmd() cmd.id = fw_rule.id self.userapiclient.deleteIpv6FirewallRule(cmd) res = self.getRouterProcessStatus(self.getNetworkRouter(self.network), routerCmd) - self.assertFalse(parsed_rule in res or parsed_rule_new in res, - "Firewall rule present in nft list chain failure despite delete for rule: '%s' is in '%s'" % (parsed_rule, res)) + self.assertFalse(self.isIpv6FirewallRuleListed(accepted_rules, res), + "Firewall rule present in nft list chain failure despite delete for rule: '%s' is in '%s'" % (accepted_rules, res)) def checkIpv6FirewallRule(self): traffic_type = "Ingress" @@ -727,8 +734,10 @@ def checkIpv6FirewallRule(self): icmp_type = choice(list(ICMPV6_TYPE.keys())) icmp_code = choice(list(ICMPV6_CODE_TYPE.keys())) rule = "ip6 saddr %s ip6 daddr %s %sv6 type %s %sv6 code %s accept" % (source_cidr, CIDR_IPV6_ANY, protocol, ICMPV6_TYPE[icmp_type], protocol, ICMPV6_CODE_TYPE[icmp_code]) + # nft prints the name of the code only for some ICMPv6 types (depending on its version), else the number + rule_with_code_number = "ip6 saddr %s ip6 daddr %s %sv6 type %s %sv6 code %d accept" % (source_cidr, CIDR_IPV6_ANY, protocol, ICMPV6_TYPE[icmp_type], protocol, icmp_code) self.createAndVerifyIpv6FirewallRule(traffic_type, source_cidr, None, protocol, - None, None, icmp_type, icmp_code, rule) + None, None, icmp_type, icmp_code, rule, alternative_rules=[rule_with_code_number]) action = "accept" if self.isNetworkEgressDefaultPolicyAllow(self.network): diff --git a/test/integration/smoke/test_vpc_ipv6.py b/test/integration/smoke/test_vpc_ipv6.py index bc05334a56ee..3c2fec791038 100644 --- a/test/integration/smoke/test_vpc_ipv6.py +++ b/test/integration/smoke/test_vpc_ipv6.py @@ -761,9 +761,13 @@ def verifyAclRulesInRouter(self, nic, rules, router): acl_chain = nic + ACL_CHAINS_SUFFIX[rule["traffictype"]] routerCmd = "nft list chain ip6 %s %s" % (ACL_TABLE, acl_chain) res = self.getRouterProcessStatus(router, routerCmd) - parsed_rule_new = rule["parsedrule"].replace("{ ", "").replace(" }", "") - self.assertTrue(rule["parsedrule"] in res or parsed_rule_new in res, - "Listing firewall rule with nft list chain failure for rule: '%s' is not in '%s'" % (rule["parsedrule"], res)) + accepted_rules = [rule["parsedrule"]] + rule.get("alternativerules", []) + found = False + for accepted_rule in accepted_rules: + if accepted_rule in res or accepted_rule.replace("{ ", "").replace(" }", "") in res: + found = True + self.assertTrue(found, + "Listing firewall rule with nft list chain failure for rule: '%s' is not in '%s'" % (accepted_rules, res)) def checkIpv6AclRule(self): router = self.getVpcRouter(self.vpc) @@ -794,7 +798,9 @@ def checkIpv6AclRule(self): rule["icmptype"] = choice(list(ICMPV6_TYPE.keys())) rule["icmpcode"] = choice(list(ICMPV6_CODE_TYPE.keys())) parsedrule = "ip6 daddr %s %sv6 type %s %sv6 code %s accept" % (rule["cidrlist"], rule["protocol"], ICMPV6_TYPE[rule["icmptype"]], rule["protocol"], ICMPV6_CODE_TYPE[rule["icmpcode"]]) - rules.append({"traffictype": rule["traffictype"], "parsedrule": parsedrule}) + # nft prints the name of the code only for some ICMPv6 types (depending on its version), else the number + parsedrule_with_code_number = "ip6 daddr %s %sv6 type %s %sv6 code %d accept" % (rule["cidrlist"], rule["protocol"], ICMPV6_TYPE[rule["icmptype"]], rule["protocol"], rule["icmpcode"]) + rules.append({"traffictype": rule["traffictype"], "parsedrule": parsedrule, "alternativerules": [parsedrule_with_code_number]}) self.createNetworkAclRule(rule, tier1_acl.id) self.network.replaceACLList(self.apiclient, tier1_acl.id) diff --git a/tools/appliance/systemvmtemplate/http/preseed_aarch64.cfg b/tools/appliance/systemvmtemplate/http/preseed_aarch64.cfg index 5262603deaa4..46967d175978 100644 --- a/tools/appliance/systemvmtemplate/http/preseed_aarch64.cfg +++ b/tools/appliance/systemvmtemplate/http/preseed_aarch64.cfg @@ -107,7 +107,7 @@ d-i passwd/user-default-groups string audio cdrom video admin ### Package selection tasksel tasksel/first multiselect ssh-server -d-i pkgsel/include string openssh-server ntp acpid sudo bzip2 openssl +d-i pkgsel/include string openssh-server ntpsec acpid sudo bzip2 openssl # Allowed values: none, safe-upgrade, full-upgrade d-i pkgsel/upgrade select full-upgrade d-i pkgsel/update-policy select none diff --git a/tools/appliance/systemvmtemplate/http/preseed_x86_64.cfg b/tools/appliance/systemvmtemplate/http/preseed_x86_64.cfg index 1ed0edd492f9..a9936c5d2c41 100644 --- a/tools/appliance/systemvmtemplate/http/preseed_x86_64.cfg +++ b/tools/appliance/systemvmtemplate/http/preseed_x86_64.cfg @@ -98,7 +98,7 @@ d-i passwd/user-default-groups string audio cdrom video admin ### Package selection tasksel tasksel/first multiselect ssh-server -d-i pkgsel/include string openssh-server ntp acpid sudo bzip2 openssl +d-i pkgsel/include string openssh-server ntpsec acpid sudo bzip2 openssl # Allowed values: none, safe-upgrade, full-upgrade d-i pkgsel/upgrade select full-upgrade d-i pkgsel/update-policy select none diff --git a/tools/appliance/systemvmtemplate/scripts/apt_upgrade.sh b/tools/appliance/systemvmtemplate/scripts/apt_upgrade.sh index 91a1dd3cdabc..38336dd0ac1d 100755 --- a/tools/appliance/systemvmtemplate/scripts/apt_upgrade.sh +++ b/tools/appliance/systemvmtemplate/scripts/apt_upgrade.sh @@ -32,12 +32,18 @@ function fix_tune2fs() { } function add_backports() { - sed -i '/cdrom/d' /etc/apt/sources.list - sed -i '/deb-src/d' /etc/apt/sources.list - sed -i '/backports/d' /etc/apt/sources.list - sed -i '/security/d' /etc/apt/sources.list - echo 'deb http://http.debian.net/debian bookworm-backports main' >> /etc/apt/sources.list - echo 'deb http://security.debian.org/debian-security bookworm-security main' >> /etc/apt/sources.list + # Debian 13 installs apt sources in /etc/apt/sources.list.d/debian.sources + # and /etc/apt/sources.list may not exist + if [ -f /etc/apt/sources.list ]; then + sed -i '/cdrom/d' /etc/apt/sources.list + sed -i '/deb-src/d' /etc/apt/sources.list + sed -i '/backports/d' /etc/apt/sources.list + sed -i '/security/d' /etc/apt/sources.list + fi + cat > /etc/apt/sources.list.d/backports-security.list << EOF +deb http://http.debian.net/debian trixie-backports main +deb http://security.debian.org/debian-security trixie-security main +EOF } function apt_upgrade() { diff --git a/tools/appliance/systemvmtemplate/scripts/configure_systemvm_services.sh b/tools/appliance/systemvmtemplate/scripts/configure_systemvm_services.sh index 8bc71d29c1c0..455e59915f87 100755 --- a/tools/appliance/systemvmtemplate/scripts/configure_systemvm_services.sh +++ b/tools/appliance/systemvmtemplate/scripts/configure_systemvm_services.sh @@ -19,7 +19,7 @@ set -e set -x -CLOUDSTACK_RELEASE=4.22.0 +CLOUDSTACK_RELEASE=24.0 function configure_apache2() { # Enable ssl, rewrite and auth @@ -41,7 +41,7 @@ function configure_issue() { __?.o/ Apache CloudStack SystemVM $CLOUDSTACK_RELEASE ( )# https://cloudstack.apache.org - (___(_) Debian GNU/Linux 12 \n \l + (___(_) Debian GNU/Linux 13 \n \l EOF } @@ -142,8 +142,12 @@ datasource: EOF touch /etc/cloud/cloud-init.disabled - systemctl stop cloud-init - systemctl disable cloud-init + # cloud-init.service was split into several units in newer cloud-init versions + for svc in cloud-init cloud-init-local cloud-init-main cloud-init-network cloud-config cloud-final; do + if systemctl cat ${svc}.service > /dev/null 2>&1; then + systemctl disable ${svc}.service + fi + done configure_apache2 configure_strongswan diff --git a/tools/appliance/systemvmtemplate/scripts/install_systemvm_packages.sh b/tools/appliance/systemvmtemplate/scripts/install_systemvm_packages.sh index d391b5c4e192..00ab7e194a33 100755 --- a/tools/appliance/systemvmtemplate/scripts/install_systemvm_packages.sh +++ b/tools/appliance/systemvmtemplate/scripts/install_systemvm_packages.sh @@ -66,7 +66,7 @@ function install_packages() { xenstore-utils libxenstore4 \ ipvsadm conntrackd libnetfilter-conntrack3 \ keepalived irqbalance \ - openjdk-17-jre-headless \ + openjdk-21-jre-headless \ ipcalc ipset \ iptables-persistent \ libtcnative-1 libssl-dev libapr1-dev \ @@ -77,7 +77,7 @@ function install_packages() { sharutils genisoimage \ strongswan libcharon-extra-plugins libstrongswan-extra-plugins strongswan-charon strongswan-starter \ virt-what open-vm-tools qemu-guest-agent hyperv-daemons cloud-guest-utils \ - conntrack apt-transport-https ca-certificates curl gnupg gnupg-agent software-properties-common + conntrack apt-transport-https ca-certificates curl gnupg gnupg-agent apt-get install -y python3-json-pointer python3-jsonschema cloud-init @@ -91,17 +91,14 @@ function install_packages() { fi # Install docker and containerd for CKS - curl -fsSL https://download.docker.com/linux/debian/gpg | sudo apt-key add - - apt-key fingerprint 0EBFCD88 - if [ "${arch}" == "arm64" ]; then - add-apt-repository "deb [arch=arm64] https://download.docker.com/linux/debian $(lsb_release -cs) stable" - elif [ "${arch}" == "amd64" ]; then - add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/debian $(lsb_release -cs) stable" - elif [ "${arch}" == "s390x" ]; then - add-apt-repository "deb [arch=s390x] https://download.docker.com/linux/debian $(lsb_release -cs) stable" - else - add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/debian $(lsb_release -cs) stable" + local docker_arch="${arch}" + if [[ "${arch}" != "arm64" && "${arch}" != "amd64" && "${arch}" != "s390x" ]]; then + docker_arch="amd64" fi + install -m 0755 -d /etc/apt/keyrings + curl -fsSL https://download.docker.com/linux/debian/gpg -o /etc/apt/keyrings/docker.asc + chmod a+r /etc/apt/keyrings/docker.asc + echo "deb [arch=${docker_arch} signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/debian $(. /etc/os-release && echo "${VERSION_CODENAME}") stable" > /etc/apt/sources.list.d/docker.list apt-get update ${apt_get} install containerd.io diff --git a/tools/appliance/systemvmtemplate/template-base_aarch64-target_aarch64.json b/tools/appliance/systemvmtemplate/template-base_aarch64-target_aarch64.json index f15552ca5cc4..fe1e363423e2 100644 --- a/tools/appliance/systemvmtemplate/template-base_aarch64-target_aarch64.json +++ b/tools/appliance/systemvmtemplate/template-base_aarch64-target_aarch64.json @@ -32,8 +32,8 @@ "format": "qcow2", "headless": true, "http_directory": "http", - "iso_checksum": "sha512:55ab206cd8b0da2898767c3eb6ab5ebef101e3925ec91b3b5f0a286136195b7072588f6ac2d059c545c6938978704ae78cd18d7d9d2a86a7380e46ce27ee4e7b", - "iso_url": "https://cdimage.debian.org/mirror/cdimage/archive/12.12.0/arm64/iso-cd/debian-12.12.0-arm64-netinst.iso", + "iso_checksum": "sha512:cff08e2f29c2846f0503fe57ad1085b5067f29acab444acdf7cdf5b10f2a11140a0e87828e4ffeb9828d188053690467ebe50151c142551d6053fc3288808b86", + "iso_url": "https://cdimage.debian.org/mirror/cdimage/release/13.7.0/arm64/iso-cd/debian-13.7.0-arm64-netinst.iso", "net_device": "virtio-net", "output_directory": "../dist", "qemu_binary": "qemu-system-aarch64", diff --git a/tools/appliance/systemvmtemplate/template-base_x86_64-target_aarch64.json b/tools/appliance/systemvmtemplate/template-base_x86_64-target_aarch64.json index c16bf1fb2a29..5c15d35888c3 100644 --- a/tools/appliance/systemvmtemplate/template-base_x86_64-target_aarch64.json +++ b/tools/appliance/systemvmtemplate/template-base_x86_64-target_aarch64.json @@ -31,8 +31,8 @@ "format": "qcow2", "headless": true, "http_directory": "http", - "iso_checksum": "sha512:55ab206cd8b0da2898767c3eb6ab5ebef101e3925ec91b3b5f0a286136195b7072588f6ac2d059c545c6938978704ae78cd18d7d9d2a86a7380e46ce27ee4e7b", - "iso_url": "https://cdimage.debian.org/mirror/cdimage/archive/12.12.0/arm64/iso-cd/debian-12.12.0-arm64-netinst.iso", + "iso_checksum": "sha512:cff08e2f29c2846f0503fe57ad1085b5067f29acab444acdf7cdf5b10f2a11140a0e87828e4ffeb9828d188053690467ebe50151c142551d6053fc3288808b86", + "iso_url": "https://cdimage.debian.org/mirror/cdimage/release/13.7.0/arm64/iso-cd/debian-13.7.0-arm64-netinst.iso", "net_device": "virtio-net", "output_directory": "../dist", "qemu_binary": "qemu-system-aarch64", diff --git a/tools/appliance/systemvmtemplate/template-base_x86_64-target_x86_64.json b/tools/appliance/systemvmtemplate/template-base_x86_64-target_x86_64.json index 941eea9dcd7b..967645b500d5 100644 --- a/tools/appliance/systemvmtemplate/template-base_x86_64-target_x86_64.json +++ b/tools/appliance/systemvmtemplate/template-base_x86_64-target_x86_64.json @@ -27,8 +27,8 @@ "format": "qcow2", "headless": true, "http_directory": "http", - "iso_checksum": "sha512:c93055182057dd19a334260671c7e10880541b7721ad9c8df87be47e0a11d5bbf85018350ff224ff6a5f6a68320b07e95d539cef9dc020c93966bfaa86d4b2ce", - "iso_url": "https://cdimage.debian.org/mirror/cdimage/archive/12.12.0/amd64/iso-cd/debian-12.12.0-amd64-netinst.iso", + "iso_checksum": "sha512:ef04d0276850d70e4aaec0b55003628660e9631b26e35215a70ab3c8c707294a4b12731bc73c75a932902f703a7c278310ca99df28d6b36fb06d5618d8d0f678", + "iso_url": "https://cdimage.debian.org/mirror/cdimage/release/13.7.0/amd64/iso-cd/debian-13.7.0-amd64-netinst.iso", "net_device": "virtio-net", "output_directory": "../dist", "qemuargs": [