systemvmtemplate: bump to Debian 13 'trixie' and ACS 24.0

engine/schema/pom.xml

@@ -88,8 +88,15 @@
                                 <script><![CDATA[
                                     def projectVersion = project.properties.getProperty('project.systemvm.template.version')
                                     String[] versionParts = projectVersion.tokenize('.')
-                                    project.properties.setProperty('cs.version', versionParts[0] + "." + versionParts[1])
-                                    project.properties.setProperty('patch.version', versionParts[2])
+                                    if (versionParts[0].toInteger() >= 24) {
+                                        // new versioning: directory 24, templates systemvmtemplate-24.0-...
+                                        project.properties.setProperty('cs.version', versionParts[0])
+                                        project.properties.setProperty('patch.version', versionParts[1])
+                                    } else {
+                                        // legacy versioning: directory 4.22, templates systemvmtemplate-4.22.0-...
+                                        project.properties.setProperty('cs.version', versionParts[0] + "." + versionParts[1])
+                                        project.properties.setProperty('patch.version', versionParts[2])
+                                    }
                                 ]]></script>
                             </scripts>
                         </configuration>

engine/schema/templateConfig.sh

@@ -28,9 +28,7 @@ function getTemplateVersion() {
   subversion2="$(cut -d'.' -f2 <<<"$version")"
   minorversion="$(cut -d'.' -f3 <<<"$version")"
   fourthversion="$(cut -d'.' -f4 <<<"$version")"
-  export CS_VERSION="${subversion1}"."${subversion2}"
   export CS_MINOR_VERSION="${minorversion}"
-  export VERSION="${CS_VERSION}.${CS_MINOR_VERSION}"
   if [[ "$subversion1" -ge "$NEW_VERSIONING_CUTOVER_MAJOR_VERSION" ]]; then
     # New versioning (major.minor.security): the third component is the
     # security release itself, there is no separate patch component. A
@@ -40,11 +38,17 @@ function getTemplateVersion() {
       echo "Invalid version '${projVersion}': major versions at or above ${NEW_VERSIONING_CUTOVER_MAJOR_VERSION} do not support a 4-component major.minor.patch.security format" >&2
       exit 1
     fi
-    export FULL_VERSION="${VERSION}"
+    # The templates are in the directory named after the major version (24),
+    # and the file names contain major.minor (24.0), like 4.22 and 4.22.0 before
+    export CS_VERSION="${subversion1}"
+    export VERSION="${subversion1}.${subversion2}"
+    export FULL_VERSION="${subversion1}.${subversion2}.${minorversion}"
   else
     # Legacy versioning: major.minor.patch, or major.minor.patch.security when a 4th
     # component is present. Without a 4th component, VERSION is already correct as-is;
     # appending an empty security component would leave a dangling trailing dot.
+    export CS_VERSION="${subversion1}"."${subversion2}"
+    export VERSION="${CS_VERSION}.${CS_MINOR_VERSION}"
     if [[ -n "$fourthversion" ]]; then
       export FULL_VERSION="${VERSION}.${fourthversion}"
     else

engine/schema/test_templateConfig.sh

@@ -51,6 +51,7 @@ assert_eq "legacy: VERSION" "4.22.0" "$VERSION"
 assert_eq "legacy: FULL_VERSION keeps the 4th (security) component" "4.22.0.0" "$FULL_VERSION"
 
 getTemplateVersion "4.23.0.1-SNAPSHOT"
+assert_eq "legacy with -SNAPSHOT suffix: CS_VERSION" "4.23" "$CS_VERSION"
 assert_eq "legacy with -SNAPSHOT suffix: VERSION" "4.23.0" "$VERSION"
 assert_eq "legacy with -SNAPSHOT suffix: FULL_VERSION" "4.23.0.1" "$FULL_VERSION"
 
@@ -62,16 +63,18 @@ assert_eq "legacy 3-component (no security): FULL_VERSION has no trailing dot" "
 # --- getTemplateVersion: new versioning (major >= 24, post-cutover) ---
 
 getTemplateVersion "24.0.0"
-assert_eq "cutover: CS_VERSION" "24.0" "$CS_VERSION"
-assert_eq "cutover: VERSION" "24.0.0" "$VERSION"
+assert_eq "cutover: CS_VERSION (directory) is the major version" "24" "$CS_VERSION"
+assert_eq "cutover: VERSION (file names) is major.minor" "24.0" "$VERSION"
 assert_eq "cutover: FULL_VERSION has no trailing dot (patch dropped)" "24.0.0" "$FULL_VERSION"
 
 getTemplateVersion "24.1.2-SNAPSHOT"
-assert_eq "cutover with -SNAPSHOT suffix: VERSION" "24.1.2" "$VERSION"
+assert_eq "cutover with -SNAPSHOT suffix: CS_VERSION" "24" "$CS_VERSION"
+assert_eq "cutover with -SNAPSHOT suffix: VERSION" "24.1" "$VERSION"
 assert_eq "cutover with -SNAPSHOT suffix: FULL_VERSION" "24.1.2" "$FULL_VERSION"
 
 getTemplateVersion "99.9.9"
-assert_eq "future major: VERSION" "99.9.9" "$VERSION"
+assert_eq "future major: CS_VERSION" "99" "$CS_VERSION"
+assert_eq "future major: VERSION" "99.9" "$VERSION"
 assert_eq "future major: FULL_VERSION" "99.9.9" "$FULL_VERSION"
 
 # A 4-component major.minor.patch.security value is invalid at/after the cutover (matches
@@ -96,7 +99,7 @@ run_create_metadata_file() {
   : > "$METADATAFILE"
 
   createMetadataFile
-  grep '^version = ' "$METADATAFILE" | head -1 | cut -d' ' -f3
+  grep "^${2:-version} = " "$METADATAFILE" | head -1 | cut -d' ' -f3
 
   rm -rf "$workdir"
 }
@@ -107,6 +110,26 @@ assert_eq "metadata.ini legacy 'version' line" "4.22.0.0" "$actual"
 actual="$(run_create_metadata_file "24.0.0")"
 assert_eq "metadata.ini cutover 'version' line has no trailing dot" "24.0.0" "$actual"
 
+# --- createMetadataFile: file names, directory, template name and checksum ---
+
+actual="$(run_create_metadata_file "4.22.0.0" downloadurl)"
+assert_eq "metadata.ini legacy downloadurl" "https://download.cloudstack.org/systemvm//4.22/systemvmtemplate-4.22.0-x86_64-kvm.qcow2.bz2" "$actual"
+actual="$(run_create_metadata_file "4.22.0.0" templatename)"
+assert_eq "metadata.ini legacy templatename" "systemvm-kvm-4.22.0-x86_64" "$actual"
+actual="$(run_create_metadata_file "4.22.0.0" checksum)"
+assert_eq "metadata.ini legacy checksum is found" "abc123" "$actual"
+
+actual="$(run_create_metadata_file "24.0.0" downloadurl)"
+assert_eq "metadata.ini cutover downloadurl" "https://download.cloudstack.org/systemvm//24/systemvmtemplate-24.0-x86_64-kvm.qcow2.bz2" "$actual"
+actual="$(run_create_metadata_file "24.0.0" filename)"
+assert_eq "metadata.ini cutover filename" "systemvmtemplate-24.0-x86_64-kvm.qcow2.bz2" "$actual"
+actual="$(run_create_metadata_file "24.0.0" templatename)"
+assert_eq "metadata.ini cutover templatename" "systemvm-kvm-24.0-x86_64" "$actual"
+actual="$(run_create_metadata_file "24.0.0" checksum)"
+assert_eq "metadata.ini cutover checksum is found" "abc123" "$actual"
+actual="$(run_create_metadata_file "24.0.1" downloadurl)"
+assert_eq "metadata.ini cutover security release uses the same template" "https://download.cloudstack.org/systemvm//24/systemvmtemplate-24.0-x86_64-kvm.qcow2.bz2" "$actual"
+
 echo ""
 echo "${pass} passed, ${fail} failed"
 [[ $fail -eq 0 ]]

pom.xml

@@ -50,7 +50,7 @@
         <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
         <project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
         <project.systemvm.template.location>https://download.cloudstack.org/systemvm</project.systemvm.template.location>
-        <project.systemvm.template.version>4.22.0.0</project.systemvm.template.version>
+        <project.systemvm.template.version>24.0.0</project.systemvm.template.version>
         <sonar.organization>apache</sonar.organization>
         <sonar.host.url>https://sonarcloud.io</sonar.host.url>
         <sonar.exclusions>engine/schema/src/main/java/org/apache/cloudstack/backup/BackupOfferingDetailsVO.java</sonar.exclusions>

scripts/installer/export-templates.sh

@@ -26,8 +26,8 @@ TEMPLATE_VERSION=$(awk -F "=" '/version/ {print $2}' ${IMAGE_PATH}${METADATA_FIL
 TEMPLATE_PATH="/usr/share/cloudstack-management/templates/systemvm/"
 TEMPLATE_MAJOR_VERSION="$(cut -d'.' -f1 <<<"$TEMPLATE_VERSION")"
 if [[ "$TEMPLATE_MAJOR_VERSION" -ge "$NEW_VERSIONING_CUTOVER_MAJOR_VERSION" ]]; then
-  # New versioning (major.minor.security): the generated template files use the full version as-is.
-  VERSION="${TEMPLATE_VERSION}"
+  # New versioning (major.minor.security): the template files contain major.minor, without the security component.
+  VERSION="$(cut -d'.' -f1,2 <<<"$TEMPLATE_VERSION")"
 else
   # Legacy versioning drops the trailing security component only for four-component metadata;
   # a valid three-component major.minor.patch value is already the filename version.

systemvm/debian/opt/cloud/bin/passwd_server_ip.py

@@ -26,14 +26,15 @@
 #   curl --header 'DomU_Request: save_password' http://localhost:8080/ -F ip=<IP> -F password=<passwd>
 
 import binascii
-import cgi
+import email.policy
 import os
 import socketserver
 import sys
 import syslog
 import threading
 import urllib.parse
 
+from email.parser import BytesParser
 from http.server import BaseHTTPRequestHandler, HTTPServer
 from socketserver import ThreadingMixIn
 
@@ -44,6 +45,26 @@
 allowAddresses = ['localhost', '127.0.0.1']
 lock = threading.RLock()
 
+
+def parseForm(contentType, body):
+    """ Parse a POST body (multipart/form-data, as sent by curl -F, or
+    application/x-www-form-urlencoded) to a dict of field name -> value.
+    The cgi module, which was used before, is not available in Python 3.13 """
+    form = {}
+    contentType = contentType or ''
+    if contentType.lower().startswith('multipart/form-data'):
+        message = BytesParser(policy=email.policy.HTTP).parsebytes(
+            b'Content-Type: ' + contentType.encode() + b'\r\n\r\n' + body)
+        if message.is_multipart():
+            for part in message.iter_parts():
+                name = part.get_param('name', header='content-disposition')
+                if name:
+                    form[name] = part.get_payload(decode=True).decode('utf-8', 'replace')
+    else:
+        for name, values in urllib.parse.parse_qs(body.decode('utf-8', 'replace'), keep_blank_values=True).items():
+            form[name] = values[0]
+    return form
+
 def getTokenFile():
     return '/tmp/passwdsrvrtoken'
 
@@ -140,12 +161,8 @@ def do_GET(self):
         return
 
     def do_POST(self):
-        form = cgi.FieldStorage(
-                    fp=self.rfile,
-                    headers=self.headers,
-                    environ={'REQUEST_METHOD':'POST',
-                             'CONTENT_TYPE':self.headers['content-type'],
-                    })
+        form = parseForm(self.headers.get('content-type'),
+                         self.rfile.read(int(self.headers.get('content-length', 0))))
         self.send_response(200)
         self.end_headers()
         clientAddress = self.client_address[0]
@@ -157,13 +174,13 @@ def do_POST(self):
             syslog.syslog('serve_password: request trying to save password does not contain both ip and password')
             self.send_response(403)
             return
-        token = form['token'].value
+        token = form['token']
         if not checkToken(token):
             syslog.syslog('serve_password: invalid save_password token received from %s' % clientAddress)
             self.send_response(403)
             return
-        ip = form['ip'].value
-        password = form['password'].value
+        ip = form['ip']
+        password = form['password']
         if not ip or not password:
             syslog.syslog('serve_password: empty ip/password[%s/%s] received from savepassword' % (ip, password))
             return

systemvm/debian/opt/cloud/bin/setup/bootstrap.sh

@@ -104,12 +104,29 @@ config_sysctl() {
   sysctl -p
 }
 
+# Debian 13 and later does not install /etc/iproute2/rt_tables, which the scripts of all the
+# systemvm types use to add the route tables
+config_rt_tables() {
+  if [ ! -f /etc/iproute2/rt_tables ]; then
+    log_it "Creating /etc/iproute2/rt_tables"
+    mkdir -p /etc/iproute2
+    if [ -f /etc/iptables/rt_tables_init ]; then
+      cp -f /etc/iptables/rt_tables_init /etc/iproute2/rt_tables
+    elif [ -f /usr/share/iproute2/rt_tables ]; then
+      cp -f /usr/share/iproute2/rt_tables /etc/iproute2/rt_tables
+    else
+      touch /etc/iproute2/rt_tables
+    fi
+  fi
+}
+
 bootstrap() {
   log_it "Bootstrapping systemvm appliance"
 
   export TYPE=$(grep -Po 'type=\K[a-zA-Z]*' $CMDLINE)
   patch
   config_sysctl
+  config_rt_tables
 
   log_it "Configuring systemvm type=$TYPE"
   if [ -f "/opt/cloud/bin/setup/$TYPE.sh" ]; then

systemvm/debian/opt/cloud/bin/setup/cksnode.sh

@@ -68,12 +68,9 @@ setup_k8s_node() {
     log_it "Starting cloud-init services"
     systemctl enable --now --no-block containerd
     if [ -f /home/cloud/success ]; then
-      systemctl stop cloud-init cloud-config cloud-final
-      systemctl disable cloud-init cloud-config cloud-final
+      stop_cloud_init
     else
-      systemctl start --no-block cloud-init
-      systemctl start --no-block cloud-config
-      systemctl start --no-block cloud-final
+      start_cloud_init
     fi
 }
 

systemvm/debian/opt/cloud/bin/setup/common.sh

@@ -700,6 +700,34 @@ setup_ntp() {
     fi
 }
 
+# cloud-init is disabled in the template with /etc/cloud/cloud-init.disabled, and the units of
+# cloud-init 25.1 and later (Debian 13) are skipped by systemd while that file exists. The generator
+# of cloud-init decides at boot, so it has to run again after the file is removed.
+# cloud-init 25.1 and later also does not have cloud-init.service anymore: cloud-init-main is
+# a single process which runs all the stages, and the units of the stages hand over to it
+start_cloud_init() {
+  rm -f /etc/cloud/cloud-init.disabled /run/cloud-init/disabled
+  systemctl daemon-reload
+  if systemctl cat cloud-init-main.service > /dev/null 2>&1; then
+    systemctl start --no-block cloud-init-main cloud-init-local cloud-init-network cloud-config cloud-final
+  else
+    systemctl start --no-block cloud-init
+    systemctl start --no-block cloud-config
+    systemctl start --no-block cloud-final
+  fi
+}
+
+stop_cloud_init() {
+  local svc
+  for svc in cloud-init cloud-init-main cloud-init-local cloud-init-network cloud-config cloud-final; do
+    if systemctl cat ${svc}.service > /dev/null 2>&1; then
+      systemctl stop ${svc}
+      systemctl disable ${svc}
+    fi
+  done
+  touch /etc/cloud/cloud-init.disabled
+}
+
 routing_svcs() {
    echo "haproxy apache2 frr" > /var/cache/cloud/enabled_svcs
    echo "cloud nfs-common portmap" > /var/cache/cloud/disabled_svcs

systemvm/debian/opt/cloud/bin/setup/default.sh

@@ -23,7 +23,6 @@ setup_default() {
 auto lo
 iface lo inet loopback
 EOF
-  cp -f /etc/iptables/rt_tables_init /etc/iproute2/rt_tables
 }
 
 setup_default

systemvm/debian/opt/cloud/bin/setup/sharedfsvm.sh

@@ -57,12 +57,9 @@ setup_sharedfsvm() {
 
     log_it "Starting cloud-init services"
     if [ -f /home/cloud/success ]; then
-      systemctl stop cloud-init cloud-config cloud-final
-      systemctl disable cloud-init cloud-config cloud-final
+      stop_cloud_init
     else
-      systemctl start --no-block cloud-init
-      systemctl start --no-block cloud-config
-      systemctl start --no-block cloud-final
+      start_cloud_init
     fi
 }
 

systemvm/test/TestPasswdServerIp.py

@@ -0,0 +1,84 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+import shutil
+import subprocess
+import threading
+import unittest
+from http.server import BaseHTTPRequestHandler, HTTPServer
+
+import passwd_server_ip
+
+
+class TestPasswdServerIp(unittest.TestCase):
+
+    def multipart(self, fields, boundary="----boundary123"):
+        body = b""
+        for name, value in fields.items():
+            body += ("--%s\r\nContent-Disposition: form-data; name=\"%s\"\r\n\r\n" % (boundary, name)).encode()
+            body += value.encode() + b"\r\n"
+        body += ("--%s--\r\n" % boundary).encode()
+        return "multipart/form-data; boundary=%s" % boundary, body
+
+    def test_parse_multipart_form(self):
+        contentType, body = self.multipart({"ip": "10.1.1.5", "password": "Ab1&=;'\"x y", "token": "abc"})
+        form = passwd_server_ip.parseForm(contentType, body)
+        self.assertEqual({"ip": "10.1.1.5", "password": "Ab1&=;'\"x y", "token": "abc"}, form)
+
+    def test_parse_multipart_form_with_non_ascii_value(self):
+        contentType, body = self.multipart({"password": "päss"})
+        self.assertEqual("päss", passwd_server_ip.parseForm(contentType, body)["password"])
+
+    def test_parse_urlencoded_form(self):
+        form = passwd_server_ip.parseForm("application/x-www-form-urlencoded", b"ip=10.1.1.5&password=a%26b%3Dc&token=abc")
+        self.assertEqual({"ip": "10.1.1.5", "password": "a&b=c", "token": "abc"}, form)
+
+    def test_parse_form_without_fields(self):
+        self.assertEqual({}, passwd_server_ip.parseForm(None, b""))
+        contentType, body = self.multipart({})
+        self.assertEqual({}, passwd_server_ip.parseForm(contentType, body))
+
+    @unittest.skipUnless(shutil.which("curl"), "curl is not available")
+    def test_parse_form_posted_by_curl(self):
+        """ the same request as /opt/cloud/bin/savepassword.sh """
+        received = {}
+
+        class Handler(BaseHTTPRequestHandler):
+            def do_POST(self):
+                length = int(self.headers.get('content-length', 0))
+                received.update(passwd_server_ip.parseForm(self.headers.get('content-type'), self.rfile.read(length)))
+                self.send_response(200)
+                self.end_headers()
+
+            def log_message(self, format, *args):
+                return
+
+        server = HTTPServer(("127.0.0.1", 0), Handler)
+        thread = threading.Thread(target=server.serve_forever)
+        thread.start()
+        try:
+            subprocess.check_call(["curl", "-s", "--header", "DomU_Request: save_password",
+                                   "http://127.0.0.1:%d/" % server.server_address[1],
+                                   "-F", "ip=10.1.1.5", "-F", "password=Pa$$w&rd=1", "-F", "token=abc"])
+        finally:
+            server.shutdown()
+            thread.join()
+            server.server_close()
+        self.assertEqual({"ip": "10.1.1.5", "password": "Pa$$w&rd=1", "token": "abc"}, received)
+
+
+if __name__ == '__main__':
+    unittest.main()