KVM Agent config to reserve dom0 CPUs (#7987)
This PR allows an admin to reserve some hypervisor host CPUs for system use. Another way to think of it is limiting the number of CPUs allocatable to VMs. This can be useful if the admin wants to do other things with the hypervisor's CPU, for example reserve some cores for running hyperconverged storage processes.
Co-authored-by: Marcus Sorensen <mls@apple.com>
diff --git a/agent/conf/agent.properties b/agent/conf/agent.properties
index 9174da7..b88da36 100644
--- a/agent/conf/agent.properties
+++ b/agent/conf/agent.properties
@@ -279,6 +279,11 @@
# If this parameter is used, property host.overcommit.mem.mb must be set to 0.
#host.reserved.mem.mb=1024
+# Number of CPU cores to subtract from advertised available cores.
+# These are reserved for system activity, or otherwise share host CPU resources with
+# CloudStack VM allocation.
+# host.reserved.cpu.count = 0
+
# The model of Watchdog timer to present to the Guest.
# For all models refer to the libvirt documentation.
#vm.watchdog.model=i6300esb
diff --git a/agent/src/main/java/com/cloud/agent/properties/AgentProperties.java b/agent/src/main/java/com/cloud/agent/properties/AgentProperties.java
index 75248fb..610c5be 100644
--- a/agent/src/main/java/com/cloud/agent/properties/AgentProperties.java
+++ b/agent/src/main/java/com/cloud/agent/properties/AgentProperties.java
@@ -503,6 +503,15 @@
public static final Property<Integer> HOST_RESERVED_MEM_MB = new Property<>("host.reserved.mem.mb", 1024);
/**
+ * How many host CPUs to reserve for non-allocation.<br>
+ * This can be used to set aside CPU cores on the host for other tasks, such as running hyperconverged storage<br>
+ * processes, etc.
+ * Data type: Integer.<br>
+ * Default value: <code>0</code>
+ */
+ public static final Property<Integer> HOST_RESERVED_CPU_CORE_COUNT = new Property<>("host.reserved.cpu.count", 0);
+
+ /**
* The model of Watchdog timer to present to the Guest.<br>
* For all models refer to the libvirt documentation.<br>
* Data type: String.<br>
diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java
index 16cccb0..6bbafca 100644
--- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java
+++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java
@@ -460,6 +460,8 @@
private long dom0OvercommitMem;
+ private int dom0MinCpuCores;
+
protected int cmdsTimeout;
protected int stopTimeout;
protected CPUStat cpuStat = new CPUStat();
@@ -1063,6 +1065,8 @@
// Reserve 1GB unless admin overrides
dom0MinMem = ByteScaleUtils.mebibytesToBytes(AgentPropertiesFileHandler.getPropertyValue(AgentProperties.HOST_RESERVED_MEM_MB));
+ dom0MinCpuCores = AgentPropertiesFileHandler.getPropertyValue(AgentProperties.HOST_RESERVED_CPU_CORE_COUNT);
+
// Support overcommit memory for host if host uses ZSWAP, KSM and other memory
// compressing technologies
dom0OvercommitMem = ByteScaleUtils.mebibytesToBytes(AgentPropertiesFileHandler.getPropertyValue(AgentProperties.HOST_OVERCOMMIT_MEM_MB));
@@ -3540,7 +3544,7 @@
@Override
public StartupCommand[] initialize() {
- final KVMHostInfo info = new KVMHostInfo(dom0MinMem, dom0OvercommitMem, manualCpuSpeed);
+ final KVMHostInfo info = new KVMHostInfo(dom0MinMem, dom0OvercommitMem, manualCpuSpeed, dom0MinCpuCores);
String capabilities = String.join(",", info.getCapabilities());
if (dpdkSupport) {
@@ -3548,7 +3552,7 @@
}
final StartupRoutingCommand cmd =
- new StartupRoutingCommand(info.getCpus(), info.getCpuSpeed(), info.getTotalMemory(), info.getReservedMemory(), capabilities, hypervisorType,
+ new StartupRoutingCommand(info.getAllocatableCpus(), info.getCpuSpeed(), info.getTotalMemory(), info.getReservedMemory(), capabilities, hypervisorType,
RouterPrivateIpStrategy.HostLocal);
cmd.setCpuSockets(info.getCpuSockets());
fillNetworkInformation(cmd);
diff --git a/plugins/hypervisors/kvm/src/main/java/org/apache/cloudstack/utils/linux/KVMHostInfo.java b/plugins/hypervisors/kvm/src/main/java/org/apache/cloudstack/utils/linux/KVMHostInfo.java
index 807b254..d160cbf 100644
--- a/plugins/hypervisors/kvm/src/main/java/org/apache/cloudstack/utils/linux/KVMHostInfo.java
+++ b/plugins/hypervisors/kvm/src/main/java/org/apache/cloudstack/utils/linux/KVMHostInfo.java
@@ -48,7 +48,8 @@
private static final Logger LOGGER = Logger.getLogger(KVMHostInfo.class);
- private int cpus;
+ private int totalCpus;
+ private int allocatableCpus;
private int cpusockets;
private long cpuSpeed;
private long totalMemory;
@@ -58,16 +59,25 @@
private static String cpuInfoFreqFileName = "/sys/devices/system/cpu/cpu0/cpufreq/base_frequency";
- public KVMHostInfo(long reservedMemory, long overCommitMemory, long manualSpeed) {
+ public KVMHostInfo(long reservedMemory, long overCommitMemory, long manualSpeed, int reservedCpus) {
this.cpuSpeed = manualSpeed;
this.reservedMemory = reservedMemory;
this.overCommitMemory = overCommitMemory;
this.getHostInfoFromLibvirt();
this.totalMemory = new MemStat(this.getReservedMemory(), this.getOverCommitMemory()).getTotal();
+ this.allocatableCpus = totalCpus - reservedCpus;
+ if (allocatableCpus < 1) {
+ LOGGER.warn(String.format("Aggressive reserved CPU config leaves no usable CPUs for VMs! Total system CPUs: %d, Reserved: %d, Allocatable: %d", totalCpus, reservedCpus, allocatableCpus));
+ allocatableCpus = 0;
+ }
}
- public int getCpus() {
- return this.cpus;
+ public int getTotalCpus() {
+ return this.totalCpus;
+ }
+
+ public int getAllocatableCpus() {
+ return this.allocatableCpus;
}
public int getCpuSockets() {
@@ -189,7 +199,7 @@
if (hosts.nodes > 0) {
this.cpusockets = hosts.sockets * hosts.nodes;
}
- this.cpus = hosts.cpus;
+ this.totalCpus = hosts.cpus;
final LibvirtCapXMLParser parser = new LibvirtCapXMLParser();
parser.parseCapabilitiesXML(capabilities);
diff --git a/plugins/hypervisors/kvm/src/test/java/org/apache/cloudstack/utils/linux/KVMHostInfoTest.java b/plugins/hypervisors/kvm/src/test/java/org/apache/cloudstack/utils/linux/KVMHostInfoTest.java
index 92cece6..3b5422f 100644
--- a/plugins/hypervisors/kvm/src/test/java/org/apache/cloudstack/utils/linux/KVMHostInfoTest.java
+++ b/plugins/hypervisors/kvm/src/test/java/org/apache/cloudstack/utils/linux/KVMHostInfoTest.java
@@ -73,8 +73,36 @@
Mockito.when(conn.close()).thenReturn(0);
int manualSpeed = 500;
- KVMHostInfo kvmHostInfo = new KVMHostInfo(10, 10, manualSpeed);
+ KVMHostInfo kvmHostInfo = new KVMHostInfo(10, 10, manualSpeed, 0);
Assert.assertEquals(kvmHostInfo.getCpuSpeed(), manualSpeed);
}
}
+
+ @Test
+ public void reservedCpuCoresTest() throws Exception {
+ if (!System.getProperty("os.name").equals("Linux")) {
+ return;
+ }
+ try (MockedStatic<LibvirtConnection> ignored = Mockito.mockStatic(LibvirtConnection.class)) {
+ Connect conn = Mockito.mock(Connect.class);
+ NodeInfo nodeInfo = Mockito.mock(NodeInfo.class);
+ nodeInfo.cpus = 10;
+ String capabilitiesXml = "<capabilities></capabilities>";
+
+ Mockito.when(LibvirtConnection.getConnection()).thenReturn(conn);
+ Mockito.when(conn.nodeInfo()).thenReturn(nodeInfo);
+ Mockito.when(conn.getCapabilities()).thenReturn(capabilitiesXml);
+ Mockito.when(conn.close()).thenReturn(0);
+ int manualSpeed = 500;
+
+ KVMHostInfo kvmHostInfo = new KVMHostInfo(10, 10, 100, 2);
+ Assert.assertEquals("reserve two CPU cores", 8, kvmHostInfo.getAllocatableCpus());
+
+ kvmHostInfo = new KVMHostInfo(10, 10, 100, 0);
+ Assert.assertEquals("no reserve CPU core setting", 10, kvmHostInfo.getAllocatableCpus());
+
+ kvmHostInfo = new KVMHostInfo(10, 10, 100, 12);
+ Assert.assertEquals("Misconfigured/too large CPU reserve", 0, kvmHostInfo.getAllocatableCpus());
+ }
+ }
}
