[WIP] Debian 13 as SystemVM template and built-in template

.github/workflows/build.yml

@@ -54,5 +54,7 @@ jobs:
       - name: Install Non-OSS
         uses: ./.github/actions/install-nonoss
       - name: Noredist Build
+        env:
+          DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }}
         run: |
           mvn -B -P developer,systemvm -Dsimulator -Dnoredist clean install -T$(nproc)

.github/workflows/ci.yml

@@ -54,6 +54,8 @@ jobs:
           git status
           ipmitool -V
       - name: Build with Maven
+        env:
+          DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }}
         run: |
           mvn -B -P developer,systemvm -Dsimulator clean install -DskipTests=true -T$(nproc)
       - name: Archive artifacts
@@ -307,6 +309,8 @@ jobs:
           mkdir -p ~/.m2/repository
           tar -xzf /tmp/artifacts/m2-cloudstack.tar.gz -C ~/.m2/repository
       - name: Setup Simulator Prerequisites
+        env:
+          DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }}
         run: |
           sudo python3 -m pip install --upgrade netaddr mysql-connector-python
           python3 -m pip install --user --upgrade tools/marvin/dist/[mM]arvin-*.tar.gz
@@ -319,6 +323,8 @@ jobs:
           find . | grep "src/main/java" | sed 's/\/java\//\/java /g' | awk '{print "--sourcefiles", $1, "\\"}' | sort | uniq >> jacoco-report.sh
           echo "--xml jacoco-coverage.xml" >> jacoco-report.sh
       - name: Start CloudStack Management Server with Simulator
+        env:
+          DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }}
         run: |
           export MAVEN_OPTS="-Xmx4096m -XX:MaxMetaspaceSize=800m -Djava.security.egd=file:/dev/urandom  -javaagent:jacoco/lib/jacocoagent.jar=address=*,port=36320,output=tcpserver --add-opens=java.base/java.lang=ALL-UNNAMED --add-exports=java.base/sun.security.x509=ALL-UNNAMED --add-opens=java.base/jdk.internal.reflect=ALL-UNNAMED"
           echo -e "\nStarting simulator"
@@ -329,6 +335,8 @@ jobs:
           echo -e "\nStarting Advanced Zone DataCenter deployment"
           python3 tools/marvin/marvin/deployDataCenter.py -i setup/dev/advdualzone.cfg 2>&1 || true
       - name: Run Integration Tests with Simulator
+        env:
+          DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }}
         run: |
           mkdir -p integration-test-results/smoke/misc
           mkdir -p integration-test-results/component

.github/workflows/docker-cloudstack-simulator.yml

@@ -37,7 +37,7 @@ jobs:
     runs-on: ubuntu-24.04
     steps:
       - name: Login to Docker Registry
-        uses: docker/login-action@650006c6eb7dba73a995cc03b0b2d7f5ca915bee # v4.2.0
+        uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0
         with:
           registry: ${{ secrets.DOCKER_REGISTRY }}
           username: ${{ secrets.DOCKERHUB_USER }}

.github/workflows/main-sonar-check.yml

@@ -49,9 +49,12 @@ jobs:
       - name: Install Non-OSS
         uses: ./.github/actions/install-nonoss
       - name: Run Build and Tests with Coverage
+        env:
+          DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }}
         run: mvn -B -T$(nproc) -P developer,systemvm,quality -Dsimulator -Dnoredist clean install
       - name: Upload to SonarQube
         env:
+          DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }}
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}
         run: mvn -B -P quality org.sonarsource.scanner.maven:sonar-maven-plugin:sonar -Dsonar.projectKey=apache_cloudstack -Dsonar.branch.name=${{ github.ref_name }}

.github/workflows/rat.yml

@@ -40,6 +40,8 @@ jobs:
       - name: Install Non-OSS
         uses: ./.github/actions/install-nonoss
       - name: RAT licence checks
+        env:
+          DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }}
         run: |
           mvn -P developer,systemvm -Dsimulator -Dnoredist -pl . org.apache.rat:apache-rat-plugin:0.18:check
       - name: Rat Report

.github/workflows/sonar-check.yml

@@ -47,10 +47,13 @@ jobs:
       - name: Install Non-OSS
         uses: ./.github/actions/install-nonoss
       - name: Run Build and Tests with Coverage
+        env:
+          DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }}
         run: mvn -B -T$(nproc) -P developer,systemvm,quality -Dsimulator -Dnoredist clean install
       - name: Upload to SonarQube
         if: github.repository == 'apache/cloudstack' && github.event.pull_request.head.repo.full_name == github.repository
         env:
+          DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }}
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}
           PR_ID: ${{ github.event.pull_request.number }}

.gitignore

@@ -99,6 +99,9 @@ tools/marvin/marvin/cloudstackAPI/*
 unittest
 venv
 waf-*
+waf3-*
+
+**/.mvn/.develocity*
 
 # this ignores _all files starting with '.'. Don't do that!
 #.*

.mvn/develocity.xml

@@ -0,0 +1,50 @@
+<?xml version="1.0" encoding="UTF-8" standalone="yes" ?>
+<!--
+
+    Licensed to the Apache Software Foundation (ASF) under one
+    or more contributor license agreements.  See the NOTICE file
+    distributed with this work for additional information
+    regarding copyright ownership.  The ASF licenses this file
+    to you under the Apache License, Version 2.0 (the
+    "License"); you may not use this file except in compliance
+    with the License.  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing,
+    software distributed under the License is distributed on an
+    "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+    KIND, either express or implied.  See the License for the
+    specific language governing permissions and limitations
+    under the License.
+
+-->
+
+<develocity
+  xmlns="https://www.gradle.com/develocity-maven" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+  xsi:schemaLocation="https://www.gradle.com/develocity-maven https://www.gradle.com/schema/develocity-maven.xsd">
+  <server>
+    <url>https://develocity.apache.org</url>
+    <allowUntrusted>false</allowUntrusted>
+  </server>
+  <buildScan>
+    <backgroundBuildScanUpload>#{isFalse(env['GITHUB_ACTIONS'])}</backgroundBuildScanUpload>
+    <obfuscation>
+      <ipAddresses>#{{'0.0.0.0'}}</ipAddresses>
+    </obfuscation>
+    <publishing>
+      <onlyIf>
+        <![CDATA[authenticated]]>
+      </onlyIf>
+    </publishing>
+  </buildScan>
+  <buildCache>
+    <local>
+      <enabled>false</enabled>
+    </local>
+    <remote>
+      <enabled>false</enabled>
+      <storeEnabled>#{isTrue(env['GITHUB_ACTIONS'])}</storeEnabled>
+    </remote>
+  </buildCache>
+</develocity>

.mvn/extensions.xml

@@ -0,0 +1,34 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+    Licensed to the Apache Software Foundation (ASF) under one
+    or more contributor license agreements.  See the NOTICE file
+    distributed with this work for additional information
+    regarding copyright ownership.  The ASF licenses this file
+    to you under the Apache License, Version 2.0 (the
+    "License"); you may not use this file except in compliance
+    with the License.  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing,
+    software distributed under the License is distributed on an
+    "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+    KIND, either express or implied.  See the License for the
+    specific language governing permissions and limitations
+    under the License.
+
+-->
+
+<extensions>
+  <extension>
+    <groupId>com.gradle</groupId>
+    <artifactId>develocity-maven-extension</artifactId>
+    <version>2.6.0</version>
+  </extension>
+  <extension>
+    <groupId>com.gradle</groupId>
+    <artifactId>common-custom-user-data-maven-extension</artifactId>
+    <version>2.4.0</version>
+  </extension>
+</extensions>

.pre-commit-config.yaml

@@ -216,3 +216,10 @@ repos:
         types: [yaml]
         files: \.ya?ml$
         exclude: ^.*k8s-.*\.ya?ml$|^.github/workflows/.*\.lock\.ya?ml$
+  - repo: https://github.com/python-jsonschema/check-jsonschema
+    rev: 12e63946db2c5cfcc9030fac21c3883ba88c6ed8 # frozen: 0.38.0
+    hooks:
+      - id: check-dependabot
+        name: validate dependabot config
+        description: ensures the dependabot config file is valid
+        files: ^\.github/dependabot\.ya?ml$

PendingReleaseNotes

@@ -55,6 +55,25 @@ example.ver.1 > example.ver.2:
     continue to snapshot live.
 
 4.23.0.0 > 24.0.0:
+  * VMware-to-KVM import using VDDK can import powered-off VMware VMs directly
+    into Ceph RBD primary storage when forceconverttopool is enabled and the
+    selected KVM conversion host supports qemu RBD access and in-place virt-v2v
+    finalization. Hosts without in-place finalization support can still use the
+    staged VDDK flow, converting to temporary qcow2 storage before copying the
+    finalized disks into RBD.
+
+  * Linstor primary storage can now be the destination for VMware-to-KVM
+    imports in all three modes. Staged imports (virt-v2v to a temporary NFS
+    location) copy the converted disks into pre-created DRBD block devices
+    through the Linstor storage adaptor; the import host must be a LINSTOR
+    satellite connected to the destination pool. Direct VDDK imports with
+    forceconverttopool write each powered-off VMware disk over nbdkit/VDDK
+    straight into the DRBD device and finalize it with virt-v2v in place.
+    VMware CBT warm migrations replicate the initial full copy and all
+    incremental changed-block cycles directly into Linstor volumes and
+    finalize in place at cutover; the conversion host requires VDDK,
+    in-place virt-v2v support and access to the Linstor pool.
+
   * KVM/Ceph: RBD volumes can now be encrypted at rest using native librbd
     LUKS2 (<encryption format='luks2' engine='librbd'>), for both data disks
     and root disks. Encryption is transparent to the guest and reuses the

agent/src/main/java/com/cloud/agent/properties/AgentProperties.java

@@ -848,6 +848,19 @@ public Property<Integer> getWorkers() {
      */
     public static final Property<String> VDDK_TRANSPORTS = new Property<>("vddk.transports", null, String.class);
 
+    /**
+     * NBD-transport compression used between the VDDK and the VMware server for the
+     * agent-driven nbdkit full-disk copies (cold direct import and CBT initial full
+     * sync). Passed to the nbdkit vddk plugin as <code>compression=&lt;value&gt;</code>.
+     * Valid values: <code>none</code>, <code>zlib</code>, <code>fastlz</code>,
+     * <code>skipz</code> (nbdkit &ge; 1.24, vSphere &ge; 6.5). <code>fastlz</code>
+     * compresses both zero regions and real data at low CPU cost; <code>skipz</code>
+     * only elides zero blocks. Does not affect the staged virt-v2v conversion path.
+     * Data type: String.<br>
+     * Default value: <code>fastlz</code>
+     */
+    public static final Property<String> VDDK_NBD_COMPRESSION = new Property<>("vddk.nbd.compression", "fastlz", String.class);
+
     /**
      * vCenter TLS certificate thumbprint used by virt-v2v VDDK mode, passed as <code>-io vddk-thumbprint=&lt;value&gt;</code>.
      * If unset, the KVM host computes it at runtime from the vCenter endpoint.

api/src/main/java/com/cloud/agent/api/to/RemoteInstanceTO.java

@@ -38,6 +38,7 @@ public class RemoteInstanceTO implements Serializable {
     private String datacenterName;
     private String clusterName;
     private String hostName;
+    private String vmwareMoref;
 
     public RemoteInstanceTO() {
     }
@@ -65,6 +66,11 @@ public RemoteInstanceTO(String instanceName, String instancePath, String vcenter
         this.hostName = hostName;
     }
 
+    public RemoteInstanceTO(String instanceName, String instancePath, String vcenterHost, String vcenterUsername, String vcenterPassword, String datacenterName, String clusterName, String hostName, String vmwareMoref) {
+        this(instanceName, instancePath, vcenterHost, vcenterUsername, vcenterPassword, datacenterName, clusterName, hostName);
+        this.vmwareMoref = vmwareMoref;
+    }
+
     public Hypervisor.HypervisorType getHypervisorType() {
         return this.hypervisorType;
     }
@@ -100,4 +106,12 @@ public String getClusterName() {
     public String getHostName() {
         return hostName;
     }
+
+    public String getVmwareMoref() {
+        return vmwareMoref;
+    }
+
+    public void setVmwareMoref(String vmwareMoref) {
+        this.vmwareMoref = vmwareMoref;
+    }
 }

api/src/main/java/com/cloud/agent/api/to/VmwareCbtChangedBlockRangeTO.java

@@ -0,0 +1,47 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package com.cloud.agent.api.to;
+
+import java.io.Serializable;
+
+public class VmwareCbtChangedBlockRangeTO implements Serializable {
+
+    private String diskId;
+    private long startOffset;
+    private long length;
+
+    public VmwareCbtChangedBlockRangeTO() {
+    }
+
+    public VmwareCbtChangedBlockRangeTO(String diskId, long startOffset, long length) {
+        this.diskId = diskId;
+        this.startOffset = startOffset;
+        this.length = length;
+    }
+
+    public String getDiskId() {
+        return diskId;
+    }
+
+    public long getStartOffset() {
+        return startOffset;
+    }
+
+    public long getLength() {
+        return length;
+    }
+}